A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core][Bugfix][Perf] Introduce `MQLLMEngine` to avoid `asyncio` OH (#8157)
Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
Alexander Matveev committed
Sep 18, 2024 at 09:56 UTC
7c7714d856eee6fa94aade729b67f00584f72a4c
36 files changed
+1467
-1172
.buildkite/test-pipeline.yaml
+3
-1
@@ -43,13 +43,15 @@ steps:
43
fast_check: true
44
source_file_dependencies:
45
- vllm/
46
+ - tests/mq_llm_engine
47
- tests/async_engine
48
- tests/test_inputs
49
- tests/multimodal
50
- tests/test_utils
51
- tests/worker
52
commands:
52
- - pytest -v -s async_engine # Async Engine
53
+ - pytest -v -s mq_llm_engine # MQLLMEngine
54
+ - pytest -v -s async_engine # AsyncLLMEngine
55
- NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
56
- pytest -v -s test_inputs.py
57
- pytest -v -s multimodal
docs/source/dev/profiling/profiling_index.rst
+2
-2
@@ -21,8 +21,8 @@ Traces can be visualized using https://ui.perfetto.dev/.
21
.. tip::
22
23
To stop the profiler - it flushes out all the profile trace files to the directory. This takes time, for example for about 100 requests worth of data for a llama 70b, it takes about 10 minutes to flush out on a H100.
24
- Set the env variable VLLM_RPC_GET_DATA_TIMEOUT_MS to a big number before you start the server. Say something like 30 minutes.
25
- ``export VLLM_RPC_GET_DATA_TIMEOUT_MS=1800000``
24
+ Set the env variable VLLM_RPC_TIMEOUT to a big number before you start the server. Say something like 30 minutes.
25
+ ``export VLLM_RPC_TIMEOUT=1800000``
26
27
Example commands and usage:
28
===========================
tests/async_engine/test_openapi_server.py
deleted
-106
@@ -1,106 +0,0 @@
1
-import openai # use the official client for correctness check
2
-import pytest
3
-import pytest_asyncio
4
-
5
-from ..utils import VLLM_PATH, RemoteOpenAIServer
6
-
7
-# any model with a chat template should work here
8
-MODEL_NAME = "facebook/opt-125m"
9
-chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
10
-assert chatml_jinja_path.exists()
11
-
12
-
13
-@pytest.fixture(scope="module")
14
-def server():
15
- args = [
16
- # use half precision for speed and memory savings in CI environment
17
- "--dtype",
18
- "float16",
19
- "--max-model-len",
20
- "2048",
21
- "--enforce-eager",
22
- "--chat-template",
23
- str(chatml_jinja_path),
24
- ]
25
-
26
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
27
- yield remote_server
28
-
29
-
30
-@pytest_asyncio.fixture
31
-async def client(server):
32
- async with server.get_async_client() as async_client:
33
- yield async_client
34
-
35
-
36
-@pytest.mark.asyncio
37
-async def test_check_models(client: openai.AsyncOpenAI):
38
- models = await client.models.list()
39
- models = models.data
40
- served_model = models[0]
41
- assert served_model.id == MODEL_NAME
42
- assert all(model.root == MODEL_NAME for model in models)
43
-
44
-
45
-@pytest.mark.asyncio
46
-async def test_single_completion(client: openai.AsyncOpenAI):
47
- completion = await client.completions.create(model=MODEL_NAME,
48
- prompt="Hello, my name is",
49
- max_tokens=5,
50
- temperature=0.0)
51
-
52
- assert completion.id is not None
53
- assert len(completion.choices) == 1
54
- assert len(completion.choices[0].text) >= 5
55
- assert completion.choices[0].finish_reason == "length"
56
- assert completion.usage == openai.types.CompletionUsage(
57
- completion_tokens=5, prompt_tokens=6, total_tokens=11)
58
-
59
- # test using token IDs
60
- completion = await client.completions.create(
61
- model=MODEL_NAME,
62
- prompt=[0, 0, 0, 0, 0],
63
- max_tokens=5,
64
- temperature=0.0,
65
- )
66
- assert len(completion.choices[0].text) >= 5
67
-
68
-
69
-@pytest.mark.asyncio
70
-async def test_single_chat_session(client: openai.AsyncOpenAI):
71
- messages = [{
72
- "role": "system",
73
- "content": "you are a helpful assistant"
74
- }, {
75
- "role": "user",
76
- "content": "what is 1+1?"
77
- }]
78
-
79
- # test single completion
80
- chat_completion = await client.chat.completions.create(model=MODEL_NAME,
81
- messages=messages,
82
- max_tokens=10,
83
- logprobs=True,
84
- top_logprobs=5)
85
- assert chat_completion.id is not None
86
- assert len(chat_completion.choices) == 1
87
-
88
- choice = chat_completion.choices[0]
89
- assert choice.finish_reason == "length"
90
- assert chat_completion.usage == openai.types.CompletionUsage(
91
- completion_tokens=10, prompt_tokens=55, total_tokens=65)
92
-
93
- message = choice.message
94
- assert message.content is not None and len(message.content) >= 10
95
- assert message.role == "assistant"
96
- messages.append({"role": "assistant", "content": message.content})
97
-
98
- # test multi-turn dialogue
99
- messages.append({"role": "user", "content": "express your result in json"})
100
- chat_completion = await client.chat.completions.create(
101
- model=MODEL_NAME,
102
- messages=messages,
103
- max_tokens=10,
104
- )
105
- message = chat_completion.choices[0].message
106
- assert message.content is not None and len(message.content) >= 0
tests/entrypoints/openai/rpc/test_zmq_client.py
deleted
-120
@@ -1,120 +0,0 @@
1
-import asyncio
2
-import tempfile
3
-import unittest
4
-import unittest.mock
5
-import uuid
6
-
7
-import pytest
8
-import pytest_asyncio
9
-
10
-from vllm.engine.async_llm_engine import AsyncLLMEngine
11
-from vllm.entrypoints.openai.rpc.client import (AsyncEngineRPCClient,
12
- RPCClientClosedError)
13
-from vllm.entrypoints.openai.rpc.server import AsyncEngineRPCServer
14
-
15
-
16
-@pytest.fixture(scope="function")
17
-def tmp_socket():
18
- with tempfile.TemporaryDirectory() as td:
19
- yield f"ipc://{td}/{uuid.uuid4()}"
20
-
21
-
22
-@pytest_asyncio.fixture(scope="function")
23
-async def dummy_server(tmp_socket, monkeypatch):
24
- dummy_engine = unittest.mock.AsyncMock()
25
-
26
- def dummy_engine_builder(*args, **kwargs):
27
- return dummy_engine
28
-
29
- with monkeypatch.context() as m:
30
- m.setattr(AsyncLLMEngine, "from_engine_args", dummy_engine_builder)
31
- server = AsyncEngineRPCServer(None, None, rpc_path=tmp_socket)
32
-
33
- loop = asyncio.get_running_loop()
34
- server_task = loop.create_task(server.run_server_loop())
35
-
36
- try:
37
- yield server
38
- finally:
39
- server_task.cancel()
40
- server.cleanup()
41
-
42
-
43
-@pytest_asyncio.fixture(scope="function")
44
-async def client(tmp_socket):
45
- client = AsyncEngineRPCClient(rpc_path=tmp_socket)
46
- # Sanity check: the server is connected
47
- await client._wait_for_server_rpc()
48
-
49
- try:
50
- yield client
51
- finally:
52
- client.close()
53
-
54
-
55
-@pytest.mark.asyncio
56
-async def test_client_data_methods_use_timeouts(monkeypatch, dummy_server,
57
- client: AsyncEngineRPCClient):
58
- with monkeypatch.context() as m:
59
- # Make the server _not_ reply with a model config
60
- m.setattr(dummy_server, "get_config", lambda x: None)
61
- m.setattr(client, "_data_timeout", 10)
62
-
63
- # And ensure the task completes anyway
64
- # (client.setup() invokes server.get_config())
65
- client_task = asyncio.get_running_loop().create_task(client.setup())
66
- with pytest.raises(TimeoutError, match="Server didn't reply within"):
67
- await asyncio.wait_for(client_task, timeout=0.05)
68
-
69
-
70
-@pytest.mark.asyncio
71
-async def test_client_aborts_use_timeouts(monkeypatch, dummy_server,
72
- client: AsyncEngineRPCClient):
73
- with monkeypatch.context() as m:
74
- # Hang all abort requests
75
- m.setattr(dummy_server, "abort", lambda x: None)
76
- m.setattr(client, "_data_timeout", 10)
77
-
78
- # The client should suppress timeouts on `abort`s
79
- # and return normally, assuming the server will eventually
80
- # abort the request.
81
- client_task = asyncio.get_running_loop().create_task(
82
- client.abort("test request id"))
83
- await asyncio.wait_for(client_task, timeout=0.05)
84
-
85
-
86
-@pytest.mark.asyncio
87
-async def test_client_data_methods_reraise_exceptions(
88
- monkeypatch, dummy_server, client: AsyncEngineRPCClient):
89
- with monkeypatch.context() as m:
90
- # Make the server raise some random exception
91
- exception = RuntimeError("Client test exception")
92
-
93
- def raiser():
94
- raise exception
95
-
96
- m.setattr(dummy_server.engine, "get_model_config", raiser)
97
- m.setattr(client, "_data_timeout", 10)
98
-
99
- client_task = asyncio.get_running_loop().create_task(client.setup())
100
- # And ensure the task completes, raising the exception
101
- with pytest.raises(RuntimeError, match=str(exception)):
102
- await asyncio.wait_for(client_task, timeout=0.05)
103
-
104
-
105
-@pytest.mark.asyncio
106
-async def test_client_errors_after_closing(monkeypatch, dummy_server,
107
- client: AsyncEngineRPCClient):
108
-
109
- client.close()
110
-
111
- # Healthchecks and generate requests will fail with explicit errors
112
- with pytest.raises(RPCClientClosedError):
113
- await client.check_health()
114
- with pytest.raises(RPCClientClosedError):
115
- async for _ in client.generate(None, None, None):
116
- pass
117
-
118
- # But no-ops like aborting will pass
119
- await client.abort("test-request-id")
120
- await client.do_log_stats()
tests/entrypoints/openai/test_accuracy.py
+25
-31
@@ -18,38 +18,32 @@ TASK = "gsm8k"
18
FILTER = "exact_match,strict-match"
19
RTOL = 0.03
20
EXPECTED_VALUE = 0.58
21
+DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
22
+MORE_ARGS_LIST = [["--enable-chunked-prefill"], ["--num-scheduler-steps", "8"]]
23
24
23
-@pytest.fixture(scope="module")
24
-def server():
25
- args = [
26
- "--max-model-len", "4096", "--enable-chunked-prefill",
27
- "--disable-log-requests", "--enforce-eager"
28
- ]
29
-
30
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31
- yield remote_server
32
-
33
-
34
-@pytest.fixture(scope="module")
35
-def server_data(server):
36
- return {
37
- "url": f"{server.url_for('v1')}/completions",
38
- }
25
+@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
26
+def test_lm_eval_accuracy(more_args):
27
+ args = list(DEFAULT_ARGS)
28
+ args.extend(more_args)
29
30
+ print(f"Running with: {args}")
31
41
-def test_lm_eval_accuracy(server_data):
42
- model_args = (f"model={MODEL_NAME},"
43
- f"base_url={server_data['url']},"
44
- f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False")
45
-
46
- results = lm_eval.simple_evaluate(
47
- model="local-completions",
48
- model_args=model_args,
49
- tasks=TASK,
50
- )
51
-
52
- measured_value = results["results"][TASK][FILTER]
53
- assert (measured_value - RTOL < EXPECTED_VALUE
54
- and measured_value + RTOL > EXPECTED_VALUE
55
- ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
32
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
33
+ url = f"{remote_server.url_for('v1')}/completions"
34
+
35
+ model_args = (
36
+ f"model={MODEL_NAME},"
37
+ f"base_url={url},"
38
+ f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False")
39
+
40
+ results = lm_eval.simple_evaluate(
41
+ model="local-completions",
42
+ model_args=model_args,
43
+ tasks=TASK,
44
+ )
45
+
46
+ measured_value = results["results"][TASK][FILTER]
47
+ assert (measured_value - RTOL < EXPECTED_VALUE
48
+ and measured_value + RTOL > EXPECTED_VALUE
49
+ ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
tests/entrypoints/openai/test_chat_template.py
renamed
+1
-1
@@ -5,7 +5,7 @@ from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
5
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
6
from vllm.transformers_utils.tokenizer import get_tokenizer
7
8
-from ..utils import VLLM_PATH
8
+from ...utils import VLLM_PATH
9
10
chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
11
assert chatml_jinja_path.exists()
tests/entrypoints/openai/test_mp_api_server.py
deleted
-40
@@ -1,40 +0,0 @@
1
-import time
2
-
3
-import pytest
4
-
5
-from vllm.entrypoints.openai.api_server import build_async_engine_client
6
-from vllm.entrypoints.openai.cli_args import make_arg_parser
7
-from vllm.utils import FlexibleArgumentParser
8
-
9
-
10
-@pytest.mark.asyncio
11
-async def test_mp_crash_detection():
12
-
13
- parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
14
- parser = make_arg_parser(parser)
15
- args = parser.parse_args([])
16
- # use an invalid tensor_parallel_size to trigger the
17
- # error in the server
18
- args.tensor_parallel_size = 65536
19
-
20
- start = time.perf_counter()
21
- async with build_async_engine_client(args):
22
- pass
23
- end = time.perf_counter()
24
-
25
- assert end - start < 60, ("Expected vLLM to gracefully shutdown in <60s "
26
- "if there is an error in the startup.")
27
-
28
-
29
-@pytest.mark.asyncio
30
-async def test_mp_cuda_init():
31
- # it should not crash, when cuda is initialized
32
- # in the API server process
33
- import torch
34
- torch.cuda.init()
35
- parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
36
- parser = make_arg_parser(parser)
37
- args = parser.parse_args([])
38
-
39
- async with build_async_engine_client(args):
40
- pass
tests/entrypoints/openai/test_serving_chat.py
+3
-2
@@ -4,7 +4,7 @@ from dataclasses import dataclass
4
from unittest.mock import MagicMock
5
6
from vllm.config import MultiModalConfig
7
-from vllm.engine.async_llm_engine import AsyncLLMEngine
7
+from vllm.engine.multiprocessing.client import MQLLMEngineClient
8
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
10
from vllm.transformers_utils.tokenizer import get_tokenizer
@@ -52,8 +52,9 @@ def test_async_serving_chat_init():
52
53
54
def test_serving_chat_should_set_correct_max_tokens():
55
- mock_engine = MagicMock(spec=AsyncLLMEngine)
55
+ mock_engine = MagicMock(spec=MQLLMEngineClient)
56
mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
57
+ mock_engine.errored = False
58
59
serving_chat = OpenAIServingChat(mock_engine,
60
MockModelConfig(),
tests/entrypoints/openai/test_serving_engine.py
+2
-2
@@ -4,7 +4,7 @@ from unittest.mock import MagicMock
4
import pytest
5
6
from vllm.config import ModelConfig
7
-from vllm.engine.protocol import AsyncEngineClient
7
+from vllm.engine.protocol import EngineClient
8
from vllm.entrypoints.openai.protocol import (ErrorResponse,
9
LoadLoraAdapterRequest,
10
UnloadLoraAdapterRequest)
@@ -18,7 +18,7 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
18
19
20
async def _async_serving_engine_init():
21
- mock_engine_client = MagicMock(spec=AsyncEngineClient)
21
+ mock_engine_client = MagicMock(spec=EngineClient)
22
mock_model_config = MagicMock(spec=ModelConfig)
23
# Set the max_model_len attribute to avoid missing attribute
24
mock_model_config.max_model_len = 2048
tests/entrypoints/openai/test_shutdown.py
+1
-1
@@ -44,5 +44,5 @@ async def test_shutdown_on_engine_failure(tmp_path):
44
prompt="Hello, my name is")
45
46
# Now the server should shut down
47
- return_code = remote_server.proc.wait(timeout=3)
47
+ return_code = remote_server.proc.wait(timeout=8)
48
assert return_code is not None
tests/mq_llm_engine/__init__.py
renamed
tests/mq_llm_engine/test_abort.py
new
+67
@@ -0,0 +1,67 @@
1
+"""Test that aborting is handled properly."""
2
+
3
+import asyncio
4
+import tempfile
5
+import uuid
6
+
7
+import pytest
8
+
9
+from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
10
+from vllm.engine.arg_utils import AsyncEngineArgs
11
+
12
+MODEL = "google/gemma-1.1-2b-it"
13
+ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
14
+RAISED_ERROR = KeyError
15
+RAISED_VALUE = "foo"
16
+EXPECTED_TOKENS = 250
17
+
18
+
19
+@pytest.fixture(scope="function")
20
+def tmp_socket():
21
+ with tempfile.TemporaryDirectory() as td:
22
+ yield f"ipc://{td}/{uuid.uuid4()}"
23
+
24
+
25
+@pytest.mark.asyncio
26
+async def test_abort(tmp_socket):
27
+ with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
28
+ ipc_path=tmp_socket) as engine:
29
+
30
+ client = await engine.make_client()
31
+
32
+ request_id_to_be_aborted = "request-aborted"
33
+ request_ids_a = [f"request-a-{idx}" for idx in range(10)]
34
+ request_ids_b = [f"request-b-{idx}" for idx in range(10)]
35
+
36
+ # Requests started before one to be aborted.
37
+ tasks = []
38
+ for request_id in request_ids_a:
39
+ tasks.append(
40
+ asyncio.create_task(
41
+ generate(client, request_id, EXPECTED_TOKENS)))
42
+
43
+ # Aborted.
44
+ task_aborted = asyncio.create_task(
45
+ generate(client, request_id_to_be_aborted, EXPECTED_TOKENS))
46
+
47
+ # Requests started after one to be aborted.
48
+ for request_id in request_ids_b:
49
+ tasks.append(
50
+ asyncio.create_task(
51
+ generate(client, request_id, EXPECTED_TOKENS)))
52
+
53
+ # Actually abort.
54
+ await asyncio.sleep(0.5)
55
+ await client.abort(request_id_to_be_aborted)
56
+
57
+ # Confirm that we got all the EXPECTED tokens from the requests.
58
+ for task in tasks:
59
+ count, request_id = await task
60
+ assert count == EXPECTED_TOKENS, (
61
+ f"{request_id} generated only {count} tokens")
62
+
63
+ # Cancel task (this will hang indefinitely if not).
64
+ task_aborted.cancel()
65
+
66
+ # Shutdown.
67
+ client.close()
tests/mq_llm_engine/test_error_handling.py
new
+244
@@ -0,0 +1,244 @@
1
+"""Test that various errors are handled properly."""
2
+
3
+import asyncio
4
+import tempfile
5
+import time
6
+import uuid
7
+from unittest.mock import Mock
8
+
9
+import pytest
10
+
11
+from tests.mq_llm_engine.utils import RemoteMQLLMEngine
12
+from vllm import SamplingParams
13
+from vllm.engine.arg_utils import AsyncEngineArgs
14
+from vllm.engine.llm_engine import LLMEngine
15
+from vllm.engine.multiprocessing import MQEngineDeadError
16
+from vllm.engine.multiprocessing.engine import MQLLMEngine
17
+from vllm.entrypoints.openai.api_server import build_async_engine_client
18
+from vllm.entrypoints.openai.cli_args import make_arg_parser
19
+from vllm.lora.request import LoRARequest
20
+from vllm.usage.usage_lib import UsageContext
21
+from vllm.utils import FlexibleArgumentParser
22
+
23
+MODEL = "google/gemma-1.1-2b-it"
24
+ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
25
+RAISED_ERROR = KeyError
26
+RAISED_VALUE = "foo"
27
+
28
+
29
+@pytest.fixture(scope="function")
30
+def tmp_socket():
31
+ with tempfile.TemporaryDirectory() as td:
32
+ yield f"ipc://{td}/{uuid.uuid4()}"
33
+
34
+
35
+def run_with_evil_forward(engine_args: AsyncEngineArgs, ipc_path: str):
36
+ # Make engine.
37
+ engine = MQLLMEngine.from_engine_args(
38
+ engine_args=engine_args,
39
+ usage_context=UsageContext.UNKNOWN_CONTEXT,
40
+ ipc_path=ipc_path)
41
+
42
+ # Raise error during first forward pass.
43
+ engine.engine.model_executor.execute_model = Mock(
44
+ side_effect=RAISED_ERROR(RAISED_VALUE))
45
+
46
+ # Run engine.
47
+ engine.start()
48
+
49
+
50
+@pytest.mark.asyncio
51
+async def test_evil_forward(tmp_socket):
52
+ with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
53
+ ipc_path=tmp_socket,
54
+ run_fn=run_with_evil_forward) as engine:
55
+
56
+ client = await engine.make_client()
57
+
58
+ # Server should be healthy after initial probe.
59
+ await asyncio.sleep(2.0)
60
+ await client.check_health()
61
+
62
+ # Throws an error in first forward pass.
63
+ with pytest.raises(RAISED_ERROR):
64
+ async for _ in client.generate(inputs="Hello my name is",
65
+ sampling_params=SamplingParams(),
66
+ request_id=uuid.uuid4()):
67
+ pass
68
+ assert client.errored
69
+
70
+ # Engine is errored, should get ENGINE_DEAD_ERROR.
71
+ with pytest.raises(MQEngineDeadError):
72
+ async for _ in client.generate(inputs="Hello my name is",
73
+ sampling_params=SamplingParams(),
74
+ request_id=uuid.uuid4()):
75
+ pass
76
+ assert client.errored
77
+
78
+ await asyncio.sleep(1.0)
79
+ with pytest.raises(RAISED_ERROR):
80
+ await client.check_health()
81
+ assert client.errored
82
+
83
+ # Shutdown.
84
+ client.close()
85
+
86
+
87
+def run_with_evil_model_executor_health(engine_args: AsyncEngineArgs,
88
+ ipc_path: str):
89
+ # Make engine.
90
+ engine = MQLLMEngine.from_engine_args(
91
+ engine_args=engine_args,
92
+ usage_context=UsageContext.UNKNOWN_CONTEXT,
93
+ ipc_path=ipc_path)
94
+
95
+ # Raise error during first forward pass.
96
+ engine.engine.model_executor.check_health = Mock(side_effect=RAISED_ERROR)
97
+
98
+ # Run engine.
99
+ engine.start()
100
+
101
+
102
+@pytest.mark.asyncio
103
+async def test_failed_health_check(tmp_socket):
104
+ with RemoteMQLLMEngine(
105
+ engine_args=ENGINE_ARGS,
106
+ ipc_path=tmp_socket,
107
+ run_fn=run_with_evil_model_executor_health) as engine:
108
+
109
+ client = await engine.make_client()
110
+ assert client.is_running
111
+
112
+ # Health probe should throw RAISED_ERROR.
113
+ await asyncio.sleep(15.)
114
+
115
+ with pytest.raises(RAISED_ERROR):
116
+ await client.check_health()
117
+ assert client.errored
118
+
119
+ # Generate call should throw ENGINE_DEAD_ERROR
120
+ with pytest.raises(MQEngineDeadError):
121
+ async for _ in client.generate(inputs="Hello my name is",
122
+ sampling_params=SamplingParams(),
123
+ request_id=uuid.uuid4()):
124
+ pass
125
+
126
+ client.close()
127
+
128
+
129
+def run_with_evil_abort(engine_args: AsyncEngineArgs, ipc_path: str):
130
+ # Make engine.
131
+ engine = MQLLMEngine.from_engine_args(
132
+ engine_args=engine_args,
133
+ usage_context=UsageContext.UNKNOWN_CONTEXT,
134
+ ipc_path=ipc_path)
135
+
136
+ # Raise error during abort call.
137
+ engine.engine.abort_request = Mock(side_effect=RAISED_ERROR)
138
+
139
+ # Run engine.
140
+ engine.start()
141
+
142
+
143
+@pytest.mark.asyncio
144
+async def test_failed_abort(tmp_socket):
145
+ with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
146
+ ipc_path=tmp_socket,
147
+ run_fn=run_with_evil_abort) as engine:
148
+
149
+ client = await engine.make_client()
150
+ assert client.is_running
151
+
152
+ # Firsh check health should work.
153
+ await client.check_health()
154
+
155
+ # Trigger an abort on the client side.
156
+ async def bad_abort_after_2s():
157
+ await asyncio.sleep(2.0)
158
+ await client.abort(request_id="foo")
159
+
160
+ # Trigger an abort in 2s from now.
161
+ abort_task = asyncio.create_task(bad_abort_after_2s())
162
+
163
+ # Exception in abort() will happen during this generation.
164
+ # This will kill the engine and should return ENGINE_DEAD_ERROR
165
+ # with reference to the original KeyError("foo")
166
+ with pytest.raises(MQEngineDeadError) as execinfo:
167
+ async for _ in client.generate(
168
+ inputs="Hello my name is",
169
+ sampling_params=SamplingParams(max_tokens=2000),
170
+ request_id=uuid.uuid4()):
171
+ pass
172
+ assert "KeyError" in repr(execinfo.value)
173
+ assert client.errored
174
+
175
+ await abort_task
176
+
177
+ # This should raise the original error.
178
+ with pytest.raises(RAISED_ERROR):
179
+ await client.check_health()
180
+
181
+ client.close()
182
+
183
+
184
+@pytest.mark.asyncio
185
+async def test_bad_request(tmp_socket):
186
+ with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
187
+ ipc_path=tmp_socket) as engine:
188
+
189
+ client = await engine.make_client()
190
+
191
+ # Invalid request should fail, but not crash the server.
192
+ with pytest.raises(ValueError):
193
+ async for _ in client.generate(inputs="Hello my name is",
194
+ sampling_params=SamplingParams(),
195
+ request_id="abcd-1",
196
+ lora_request=LoRARequest(
197
+ "invalid-lora", 1,
198
+ "invalid-path")):
199
+ pass
200
+
201
+ # This request should be okay.
202
+ async for _ in client.generate(inputs="Hello my name is",
203
+ sampling_params=SamplingParams(),
204
+ request_id="abcd-2"):
205
+ pass
206
+
207
+ # Shutdown.
208
+ client.close()
209
+
210
+
211
+@pytest.mark.asyncio
212
+async def test_mp_crash_detection(monkeypatch):
213
+
214
+ parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
215
+ parser = make_arg_parser(parser)
216
+ args = parser.parse_args([])
217
+
218
+ # When LLMEngine is loaded, it will crash.
219
+ def mock_init():
220
+ raise ValueError
221
+
222
+ monkeypatch.setattr(LLMEngine, "__init__", mock_init)
223
+
224
+ start = time.perf_counter()
225
+ async with build_async_engine_client(args):
226
+ pass
227
+ end = time.perf_counter()
228
+
229
+ assert end - start < 60, ("Expected vLLM to gracefully shutdown in <60s "
230
+ "if there is an error in the startup.")
231
+
232
+
233
+@pytest.mark.asyncio
234
+async def test_mp_cuda_init():
235
+ # it should not crash, when cuda is initialized
236
+ # in the API server process
237
+ import torch
238
+ torch.cuda.init()
239
+ parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
240
+ parser = make_arg_parser(parser)
241
+ args = parser.parse_args([])
242
+
243
+ async with build_async_engine_client(args):
244
+ pass
tests/mq_llm_engine/test_load.py
new
+57
@@ -0,0 +1,57 @@
1
+"""Test that the MQLLMEngine is able to handle 10k concurrent requests."""
2
+
3
+import asyncio
4
+import tempfile
5
+import uuid
6
+
7
+import pytest
8
+
9
+from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
10
+from vllm.engine.arg_utils import AsyncEngineArgs
11
+
12
+MODEL = "google/gemma-1.1-2b-it"
13
+NUM_EXPECTED_TOKENS = 10
14
+NUM_REQUESTS = 10000
15
+
16
+# Scenarios to test for num generated token.
17
+ENGINE_ARGS = AsyncEngineArgs(model=MODEL, disable_log_requests=True)
18
+
19
+
20
+@pytest.fixture(scope="function")
21
+def tmp_socket():
22
+ with tempfile.TemporaryDirectory() as td:
23
+ yield f"ipc://{td}/{uuid.uuid4()}"
24
+
25
+
26
+@pytest.mark.asyncio
27
+async def test_load(tmp_socket):
28
+ with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
29
+ ipc_path=tmp_socket) as engine:
30
+
31
+ client = await engine.make_client()
32
+
33
+ request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
34
+
35
+ # Create concurrent requests.
36
+ tasks = []
37
+ for request_id in request_ids:
38
+ tasks.append(
39
+ asyncio.create_task(
40
+ generate(client, request_id, NUM_EXPECTED_TOKENS)))
41
+
42
+ # Confirm that we got all the EXPECTED tokens from the requests.
43
+ failed_request_id = None
44
+ tokens = None
45
+ for task in tasks:
46
+ num_generated_tokens, request_id = await task
47
+ if (num_generated_tokens != NUM_EXPECTED_TOKENS
48
+ and failed_request_id is None):
49
+ failed_request_id = request_id
50
+ tokens = num_generated_tokens
51
+
52
+ assert failed_request_id is None, (
53
+ f"{failed_request_id} generated {tokens} but "
54
+ f"expected {NUM_EXPECTED_TOKENS}")
55
+
56
+ # Shutdown.
57
+ client.close()
tests/mq_llm_engine/utils.py
new
+78
@@ -0,0 +1,78 @@
1
+import asyncio
2
+import multiprocessing
3
+from typing import Callable, Tuple, Union
4
+
5
+from vllm import SamplingParams
6
+from vllm.engine.arg_utils import AsyncEngineArgs
7
+from vllm.engine.multiprocessing.client import MQLLMEngineClient
8
+from vllm.engine.multiprocessing.engine import MQLLMEngine
9
+from vllm.outputs import RequestOutput
10
+from vllm.usage.usage_lib import UsageContext
11
+
12
+
13
+async def generate(
14
+ client: MQLLMEngineClient,
15
+ request_id: str,
16
+ num_tokens: int,
17
+ return_output: bool = False) -> Union[RequestOutput, Tuple[int, str]]:
18
+
19
+ final_output = None
20
+ count = 0
21
+ async for out in client.generate(
22
+ request_id=request_id,
23
+ inputs="Hello my name is Robert and",
24
+ sampling_params=SamplingParams(max_tokens=num_tokens,
25
+ temperature=0)):
26
+
27
+ count += 1
28
+ final_output = out
29
+ await asyncio.sleep(0.)
30
+
31
+ if return_output:
32
+ return final_output
33
+
34
+ # Confirm we generated all the tokens we expected.
35
+ return count, request_id
36
+
37
+
38
+def run_normal(engine_args: AsyncEngineArgs, ipc_path: str):
39
+ # Make engine.
40
+ engine = MQLLMEngine.from_engine_args(
41
+ engine_args=engine_args,
42
+ usage_context=UsageContext.UNKNOWN_CONTEXT,
43
+ ipc_path=ipc_path)
44
+
45
+ # Run engine.
46
+ engine.start()
47
+
48
+
49
+class RemoteMQLLMEngine:
50
+
51
+ def __init__(self,
52
+ engine_args: AsyncEngineArgs,
53
+ ipc_path: str,
54
+ run_fn: Callable = run_normal) -> None:
55
+
56
+ self.engine_args = engine_args
57
+ self.ipc_path = ipc_path
58
+ context = multiprocessing.get_context("spawn")
59
+ self.proc = context.Process(target=run_fn,
60
+ args=(engine_args, ipc_path))
61
+ self.proc.start()
62
+
63
+ def __enter__(self):
64
+ return self
65
+
66
+ def __exit__(self, exc_type, exc_value, traceback):
67
+ self.proc.kill()
68
+
69
+ async def make_client(self) -> MQLLMEngineClient:
70
+ engine_config = self.engine_args.create_engine_config()
71
+ client = MQLLMEngineClient(self.ipc_path, engine_config)
72
+ while True:
73
+ try:
74
+ await client.setup()
75
+ break
76
+ except TimeoutError:
77
+ assert self.proc.is_alive()
78
+ return client
tests/tpu/test_custom_dispatcher.py
+7
@@ -1,5 +1,12 @@
1
+import os
2
+
3
from ..utils import compare_two_settings
4
5
+# --enforce-eager on TPU causes graph compilation
6
+# this times out default Health Check in the MQLLMEngine,
7
+# so we set the timeout here to 30s
8
+os.environ["VLLM_RPC_TIMEOUT"] = "30000"
9
+
10
11
def test_custom_dispatcher():
12
compare_two_settings("google/gemma-2b",
tests/utils.py
+1
-1
@@ -119,7 +119,7 @@ class RemoteOpenAIServer:
119
def __exit__(self, exc_type, exc_value, traceback):
120
self.proc.terminate()
121
try:
122
- self.proc.wait(3)
122
+ self.proc.wait(8)
123
except subprocess.TimeoutExpired:
124
# force kill if needed
125
self.proc.kill()
vllm/engine/async_llm_engine.py
+6
-3
@@ -601,9 +601,12 @@ class AsyncLLMEngine:
601
return self._errored_with is not None
602
603
@property
604
- def limit_concurrency(self) -> Optional[int]:
605
- """Maximum number of concurrently running requests."""
606
- return None
604
+ def dead_error(self) -> BaseException:
605
+ return AsyncEngineDeadError(
606
+ "Background loop is not running. If it was running, "
607
+ "inspect the output to find the stacktrace of the "
608
+ "error that caused the background loop to stop "
609
+ "(AsyncEngineDeadError).")
610
611
def set_errored(self, exc: Exception) -> None:
612
self._errored_with = exc
vllm/engine/llm_engine.py
+1
@@ -1289,6 +1289,7 @@ class LLMEngine:
1289
# torch.distributed ops which may otherwise timeout, and unblocks
1290
# the RPC thread in the workers so that they can process any other
1291
# queued control plane messages, such as add/remove lora adapters.
1292
+ logger.debug("Stopping remote worker execution loop.")
1293
self.model_executor.stop_remote_worker_execution_loop()
1294
1295
return ctx.request_outputs
vllm/engine/multiprocessing/__init__.py
new
+73
@@ -0,0 +1,73 @@
1
+from dataclasses import dataclass
2
+from enum import Enum
3
+from typing import List, Mapping, Optional, Union
4
+
5
+from vllm.inputs import PromptInputs
6
+from vllm.lora.request import LoRARequest
7
+from vllm.outputs import RequestOutput
8
+from vllm.prompt_adapter.request import PromptAdapterRequest
9
+from vllm.sampling_params import SamplingParams
10
+
11
+VLLM_RPC_SUCCESS_STR = "SUCCESS"
12
+
13
+IPC_INPUT_EXT = "_input_socket"
14
+IPC_OUTPUT_EXT = "_output_socket"
15
+IPC_HEALTH_EXT = "_health_socket"
16
+IPC_DATA_EXT = "_data_socket"
17
+
18
+
19
+class MQEngineDeadError(RuntimeError):
20
+ pass
21
+
22
+
23
+@dataclass
24
+class RPCGenerateRequest:
25
+ inputs: PromptInputs
26
+ sampling_params: SamplingParams
27
+ request_id: str
28
+ lora_request: Optional[LoRARequest] = None
29
+ trace_headers: Optional[Mapping[str, str]] = None
30
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
31
+
32
+
33
+@dataclass
34
+class RPCError:
35
+ request_id: Optional[str]
36
+ is_engine_errored: bool
37
+ exception: BaseException
38
+
39
+
40
+@dataclass
41
+class RPCAbortRequest:
42
+ request_id: str
43
+
44
+
45
+class RPCHealthRequest:
46
+ pass
47
+
48
+
49
+class RPCStartupRequest(Enum):
50
+ IS_SERVER_READY = 1
51
+
52
+
53
+@dataclass
54
+class RPCStartupResponse:
55
+ tracing_enabled: bool
56
+
57
+
58
+RPC_REQUEST_T = Union[RPCGenerateRequest, RPCAbortRequest, RPCHealthRequest,
59
+ RPCStartupRequest]
60
+
61
+REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
62
+
63
+
64
+def ENGINE_DEAD_ERROR(
65
+ error: Optional[BaseException] = None) -> MQEngineDeadError:
66
+ if error is None:
67
+ return MQEngineDeadError(
68
+ "Engine loop is not running. Inspect the stacktrace to "
69
+ "find the original error")
70
+
71
+ return MQEngineDeadError(
72
+ "Engine loop is not running. Inspect the stacktrace to "
73
+ f"find the original error: {repr(error)}.")
vllm/engine/multiprocessing/client.py
new
+452
@@ -0,0 +1,452 @@
1
+import asyncio
2
+import copy
3
+import pickle
4
+from contextlib import contextmanager, suppress
5
+from typing import (Any, AsyncGenerator, Dict, Iterator, Mapping, Optional,
6
+ Union)
7
+
8
+import cloudpickle
9
+import zmq
10
+import zmq.asyncio
11
+from zmq import Frame # type: ignore[attr-defined]
12
+from zmq.asyncio import Socket
13
+
14
+from vllm.config import DecodingConfig, EngineConfig, ModelConfig
15
+from vllm.engine.arg_utils import AsyncEngineArgs
16
+# yapf conflicts with isort for this block
17
+# yapf: disable
18
+from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
19
+ IPC_HEALTH_EXT, IPC_INPUT_EXT,
20
+ IPC_OUTPUT_EXT, RPC_REQUEST_T,
21
+ VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
22
+ RPCError, RPCGenerateRequest,
23
+ RPCHealthRequest, RPCStartupRequest,
24
+ RPCStartupResponse)
25
+# yapf: enable
26
+from vllm.envs import VLLM_RPC_TIMEOUT
27
+from vllm.inputs import PromptInputs
28
+from vllm.logger import init_logger
29
+from vllm.lora.request import LoRARequest
30
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
31
+from vllm.prompt_adapter.request import PromptAdapterRequest
32
+from vllm.sampling_params import SamplingParams
33
+from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
34
+
35
+logger = init_logger(__name__)
36
+
37
+
38
+class MQClientClosedError(Exception):
39
+ """Exception class raised when the client is used post-close.
40
+
41
+ The client can be closed, which closes the ZMQ context. This normally
42
+ happens on server shutdown. In some cases, methods like abort and
43
+ do_log_stats will still be called and then try to open a socket, which
44
+ causes a ZMQError and creates a huge stack trace.
45
+ So, we throw this error such that we can suppress it.
46
+ """
47
+
48
+
49
+class MQLLMEngineClient:
50
+ """A client wrapper for MQLLMEngine that conforms to the
51
+ EngineClient protocol.
52
+
53
+ MQLLMEngine and MQLLMEngineClient are intended to run in separate
54
+ processes communicating via zeromq ipc sockets.
55
+
56
+ The entrypoint to MQLLMEngineClient is through the generate()
57
+ method. On generate() MQLLMEngine does three things:
58
+ - Creates an asyncio output queue
59
+ - Sends a RPCGenerateRequest to the MQLLMEngine via zmq
60
+ - Pulls RequestOutputs from its queue and yields them
61
+
62
+ MQLLMEngine runs two background loops:
63
+ - output_loop: the output loop pulls List[RequestOutput]
64
+ from the MQLLMEngine via zmq (each list is the output
65
+ of one engine_step in the LLMEngine). It then parses
66
+ the list and pushes individual request_outputs into
67
+ the corresponding output_queue such that they can be
68
+ consumed by the .generate() method.
69
+ - health_loop: the health loop queries the health socket
70
+ every N seconds, confirming the engine is healthy
71
+ """
72
+
73
+ def __init__(self, ipc_path: str, engine_config: EngineConfig):
74
+ self.context = zmq.asyncio.Context()
75
+ self._errored_with: Optional[BaseException] = None
76
+
77
+ # Get the configs.
78
+ self.model_config = engine_config.model_config
79
+ self.decoding_config = engine_config.decoding_config
80
+
81
+ # Create the tokenizer group.
82
+ self.tokenizer = init_tokenizer_from_configs(
83
+ model_config=self.model_config,
84
+ scheduler_config=engine_config.scheduler_config,
85
+ parallel_config=engine_config.parallel_config,
86
+ enable_lora=bool(engine_config.lora_config),
87
+ )
88
+
89
+ # Send RPCGenerateRequest to the MQLLMEngine.
90
+ self.input_socket: Socket = self.context.socket(zmq.constants.PUSH)
91
+ self.input_socket.connect(f"{ipc_path}{IPC_INPUT_EXT}")
92
+
93
+ # Receive streams of RequestOutput from the MQLLMEngine.
94
+ self.output_socket: Socket = self.context.socket(zmq.constants.PULL)
95
+ self.output_socket.connect(f"{ipc_path}{IPC_OUTPUT_EXT}")
96
+
97
+ # IPC path for ack of check_health requests.
98
+ self.health_socket: Socket = self.context.socket(zmq.constants.PULL)
99
+ self.health_socket.connect(f"{ipc_path}{IPC_HEALTH_EXT}")
100
+
101
+ # IPC path for the data socket.
102
+ self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
103
+
104
+ # Stream for each individual request.
105
+ self.output_queues: Dict[str, asyncio.Queue] = {}
106
+ self.output_loop = asyncio.create_task(self.run_output_handler_loop())
107
+
108
+ # Loop to check health of the LLMEngine periodically.
109
+ # Started after the MQLLMEngine is ready.
110
+ self.health_loop: Optional[asyncio.Task] = None
111
+
112
+ @staticmethod
113
+ def is_unsupported_config(engine_args: AsyncEngineArgs):
114
+ if engine_args.pipeline_parallel_size > 1:
115
+ return True
116
+
117
+ is_embedding = ModelConfig(
118
+ model=engine_args.model,
119
+ revision=engine_args.revision,
120
+ tokenizer=engine_args.model,
121
+ tokenizer_mode="auto",
122
+ trust_remote_code=engine_args.trust_remote_code,
123
+ quantization=engine_args.quantization,
124
+ seed=0,
125
+ dtype="auto").embedding_mode
126
+
127
+ return is_embedding
128
+
129
+ @contextmanager
130
+ def get_data_socket(self) -> Iterator[Socket]:
131
+ socket = self.context.socket(zmq.constants.DEALER)
132
+ try:
133
+ socket.connect(self.data_ipc_path)
134
+ yield socket
135
+ finally:
136
+ socket.close(linger=0)
137
+
138
+ async def run_check_health_loop(self, timeout: int):
139
+ """Background loop that continually probes the RPCServer for health.
140
+
141
+ The loop sends CHECK_HEALTH requests to the INPUT_SOCKET, which
142
+ the MQLLMEngine server is blocking on.
143
+
144
+ The Server replies on the HEALTH_SOCKET (rather than on the
145
+ OUTPUT_SOCKET such that the messages are not intermingled with
146
+ output streaming).
147
+ """
148
+
149
+ try:
150
+ while True:
151
+ if await self.health_socket.poll(timeout=timeout) == 0:
152
+ # Wakeup every N seconds and do a health probe.
153
+ await self._send_one_way_rpc_request(
154
+ RPCHealthRequest(), self.input_socket)
155
+
156
+ # Wait for ack from the health socket.
157
+ await self._await_ack(error_message="Health check failed.",
158
+ socket=self.health_socket)
159
+ else:
160
+ # Server sent a health status message unprompted.
161
+ await self._check_success(
162
+ error_message="Health check failed.",
163
+ socket=self.health_socket)
164
+
165
+ logger.debug("Health probe successful.")
166
+
167
+ except asyncio.CancelledError:
168
+ logger.debug("Shutting down MQLLMEngineClient check health loop.")
169
+
170
+ except Exception as e:
171
+ self._set_errored(e)
172
+
173
+ async def run_output_handler_loop(self):
174
+ """Get RequestOutputs from Engine and stream to Request Queues"""
175
+
176
+ try:
177
+ while True:
178
+ # Poll, checking for ENGINE_DEAD
179
+ while await self.output_socket.poll(timeout=VLLM_RPC_TIMEOUT
180
+ ) == 0:
181
+ logger.debug("Waiting for output from MQLLMEngine.")
182
+
183
+ # If errored, alert all running requests.
184
+ if self.errored:
185
+ for queue_j in tuple(self.output_queues.values()):
186
+ queue_j.put_nowait(
187
+ ENGINE_DEAD_ERROR(self._errored_with))
188
+ return
189
+
190
+ message: Frame = await self.output_socket.recv(copy=False)
191
+ request_outputs = pickle.loads(message.buffer)
192
+
193
+ is_error = isinstance(request_outputs,
194
+ (BaseException, RPCError))
195
+ if is_error:
196
+ if isinstance(request_outputs, RPCError):
197
+ rpc_error: RPCError = request_outputs
198
+ request_id = rpc_error.request_id
199
+ exception = rpc_error.exception
200
+ is_engine_errored = rpc_error.is_engine_errored
201
+ else:
202
+ # MPLLMEngine should always return an RPCError to
203
+ # the output_socket when an issue arises.
204
+ # If we are here, we are in a bad state and
205
+ # should shut down the server.
206
+ error: BaseException = request_outputs
207
+ logger.error(
208
+ "Received Exception %s rather than RPCError from "
209
+ "MPLLMEngine. This should never happen.", error)
210
+ request_id = None
211
+ exception = error
212
+ is_engine_errored = True
213
+
214
+ # Set to error state only on engine critical error
215
+ # (and record only the first one)
216
+ if is_engine_errored and not self._errored_with:
217
+ self._errored_with = exception
218
+
219
+ if request_id is None:
220
+ for queue_i in tuple(self.output_queues.values()):
221
+ queue_i.put_nowait(exception)
222
+ else:
223
+ queue = self.output_queues.get(request_id)
224
+ if queue is not None:
225
+ queue.put_nowait(exception)
226
+ else:
227
+ # Put each output into the appropriate steam.
228
+ for request_output in request_outputs:
229
+ queue = self.output_queues.get(
230
+ request_output.request_id)
231
+ if queue is not None:
232
+ queue.put_nowait(request_output)
233
+
234
+ except asyncio.CancelledError:
235
+ logger.debug("Shutting down MQLLMEngineClient output handler.")
236
+
237
+ async def setup(self):
238
+ """Setup the client before it starts sending server requests."""
239
+
240
+ with self.get_data_socket() as socket:
241
+ # Wait until server is ready.
242
+ response = await self._wait_for_server_rpc(socket)
243
+
244
+ self.tracing_flag = response.tracing_enabled
245
+
246
+ # Start health_loop.
247
+ self.health_loop = asyncio.create_task(
248
+ self.run_check_health_loop(timeout=VLLM_RPC_TIMEOUT))
249
+
250
+ def close(self):
251
+ """Destroy the ZeroMQ Context."""
252
+ # Close all sockets and terminate the context.
253
+ self.context.destroy(linger=0)
254
+
255
+ # Cancel background tasks.
256
+ if self.health_loop is not None:
257
+ self.health_loop.cancel()
258
+ self.output_loop.cancel()
259
+
260
+ def _set_errored(self, e: BaseException):
261
+ logger.exception(repr(e))
262
+ if self._errored_with is None:
263
+ self._errored_with = e
264
+
265
+ @staticmethod
266
+ async def _send_get_data_rpc_request(request: RPCStartupRequest,
267
+ expected_type: Any,
268
+ error_message: str,
269
+ socket: Socket) -> Any:
270
+ """Send an RPC request that is expecting data back."""
271
+
272
+ # Ping RPCServer with a request.
273
+ await socket.send_multipart((pickle.dumps(request), ), copy=False)
274
+
275
+ # Make sure the server responds in time.
276
+ if await socket.poll(timeout=VLLM_RPC_TIMEOUT) == 0:
277
+ raise TimeoutError("RPCServer didn't reply within "
278
+ f"{VLLM_RPC_TIMEOUT} ms")
279
+
280
+ # Await the data from the Server.
281
+ frame = await socket.recv(copy=False)
282
+ data = pickle.loads(frame.buffer)
283
+
284
+ if isinstance(data, BaseException):
285
+ raise data
286
+ elif not isinstance(data, expected_type):
287
+ raise ValueError(error_message)
288
+
289
+ return data
290
+
291
+ @staticmethod
292
+ async def _send_one_way_rpc_request(request: RPC_REQUEST_T,
293
+ socket: Socket):
294
+ """Send one-way RPC request to trigger an action."""
295
+
296
+ if socket.closed:
297
+ raise MQClientClosedError()
298
+
299
+ await socket.send_multipart((pickle.dumps(request), ))
300
+
301
+ async def _await_ack(self, error_message: str, socket: Socket):
302
+ """Await acknowledgement that a request succeeded."""
303
+
304
+ if socket.closed:
305
+ raise MQClientClosedError()
306
+
307
+ if await socket.poll(timeout=VLLM_RPC_TIMEOUT) == 0:
308
+ raise TimeoutError("MQLLMEngine didn't reply within "
309
+ f"{VLLM_RPC_TIMEOUT}ms")
310
+
311
+ await self._check_success(error_message, socket)
312
+
313
+ @staticmethod
314
+ async def _check_success(error_message: str, socket: Socket):
315
+ """Confirm that socket has a VLLM_RPC_SUCCESS_STR message"""
316
+
317
+ if socket.closed:
318
+ raise MQClientClosedError()
319
+
320
+ frame = await socket.recv(copy=False)
321
+ response = pickle.loads(frame.buffer)
322
+
323
+ # Raise error if unsuccessful
324
+ if isinstance(response, BaseException):
325
+ raise response
326
+ elif (not isinstance(response, str)
327
+ or response != VLLM_RPC_SUCCESS_STR):
328
+ raise ValueError(error_message)
329
+
330
+ async def get_tokenizer(self, lora_request: LoRARequest):
331
+ return await self.tokenizer.get_lora_tokenizer_async(lora_request)
332
+
333
+ async def get_decoding_config(self) -> DecodingConfig:
334
+ return self.decoding_config
335
+
336
+ async def get_model_config(self) -> ModelConfig:
337
+ return self.model_config
338
+
339
+ async def is_tracing_enabled(self) -> bool:
340
+ return self.tracing_flag
341
+
342
+ async def _wait_for_server_rpc(self, socket: Socket) -> RPCStartupResponse:
343
+ """Wait for the RPCServer to start up."""
344
+
345
+ return await self._send_get_data_rpc_request(
346
+ request=RPCStartupRequest.IS_SERVER_READY,
347
+ expected_type=RPCStartupResponse,
348
+ error_message="Unable to start RPC Server",
349
+ socket=socket)
350
+
351
+ async def abort(self, request_id: str):
352
+ """Send an ABORT_REQUEST signal to the RPC Server"""
353
+
354
+ with suppress(MQClientClosedError):
355
+ await self._send_one_way_rpc_request(
356
+ request=RPCAbortRequest(request_id), socket=self.input_socket)
357
+
358
+ async def do_log_stats(self):
359
+ """Ignore do_log_stats (handled on MQLLMEngine polling)"""
360
+ pass
361
+
362
+ async def check_health(self):
363
+ """
364
+ The check health loop probes the health status of the
365
+ Engine's health every N seconds and sets _errored_with
366
+ if the engine is unhealthy.
367
+ """
368
+ if self._errored_with is not None:
369
+ raise self._errored_with
370
+
371
+ @property
372
+ def is_running(self) -> bool:
373
+ return not self.errored
374
+
375
+ @property
376
+ def is_stopped(self) -> bool:
377
+ return self.errored
378
+
379
+ @property
380
+ def errored(self) -> bool:
381
+ return self._errored_with is not None
382
+
383
+ async def generate(
384
+ self,
385
+ inputs: PromptInputs,
386
+ sampling_params: SamplingParams,
387
+ request_id: str,
388
+ lora_request: Optional[LoRARequest] = None,
389
+ trace_headers: Optional[Mapping[str, str]] = None,
390
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
391
+ ) -> AsyncGenerator[RequestOutput, None]:
392
+ """Send an RPCGenerateRequest to the RPCServer and stream responses."""
393
+
394
+ # If already dead, error out.
395
+ if self._errored_with is not None:
396
+ raise ENGINE_DEAD_ERROR(self._errored_with)
397
+
398
+ # 1) Create output queue for this requests.
399
+ queue: asyncio.Queue[Union[RequestOutput,
400
+ BaseException]] = asyncio.Queue()
401
+ self.output_queues[request_id] = queue
402
+
403
+ try:
404
+ # 2) Detach logits processors so that they can be pickled
405
+ # separately (may require cloudpickle which is slower)
406
+ if sampling_params.logits_processors:
407
+ # Defensive shallow copy
408
+ sampling_params = copy.copy(sampling_params)
409
+ logits_processors = sampling_params.logits_processors
410
+ sampling_params.logits_processors = None
411
+ lp_bytes = cloudpickle.dumps(logits_processors)
412
+ else:
413
+ lp_bytes = None
414
+
415
+ request_bytes = pickle.dumps(
416
+ RPCGenerateRequest(
417
+ inputs=inputs,
418
+ sampling_params=sampling_params,
419
+ request_id=request_id,
420
+ lora_request=lora_request,
421
+ trace_headers=trace_headers,
422
+ prompt_adapter_request=prompt_adapter_request))
423
+
424
+ # 3) Send the RPCGenerateRequest to the MQLLMEngine.
425
+ parts = (request_bytes,
426
+ lp_bytes) if lp_bytes else (request_bytes, )
427
+ await self.input_socket.send_multipart(parts, copy=False)
428
+
429
+ # 4) Stream the RequestOutputs from the output queue. Note
430
+ # that the output_loop pushes RequestOutput objects to this
431
+ # queue after pulling them from the zmq socket.
432
+ finished = False
433
+ try:
434
+ while not finished:
435
+ request_output = await queue.get()
436
+
437
+ if isinstance(request_output, BaseException):
438
+ raise request_output
439
+
440
+ finished = request_output.finished
441
+ yield request_output
442
+ finally:
443
+ # Request was canceled by the client.
444
+ if not finished and not self.errored:
445
+ await self.abort(request_id)
446
+ finally:
447
+ self.output_queues.pop(request_id)
448
+
449
+ async def encode(self, *args,
450
+ **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
451
+ raise NotImplementedError(
452
+ "Embeddings not supported with multiprocessing backend")
vllm/engine/multiprocessing/engine.py
new
+321
@@ -0,0 +1,321 @@
1
+import pickle
2
+import signal
3
+from contextlib import contextmanager
4
+from typing import Iterator, List, Optional, Union
5
+
6
+import cloudpickle
7
+import zmq
8
+
9
+from vllm import AsyncEngineArgs, LLMEngine
10
+from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11
+ ParallelConfig, SchedulerConfig)
12
+# yapf conflicts with isort for this block
13
+# yapf: disable
14
+from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
15
+ IPC_HEALTH_EXT, IPC_INPUT_EXT,
16
+ IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
17
+ VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
18
+ RPCError, RPCGenerateRequest,
19
+ RPCHealthRequest, RPCStartupRequest,
20
+ RPCStartupResponse)
21
+# yapf: enable
22
+from vllm.logger import init_logger
23
+from vllm.outputs import RequestOutput
24
+from vllm.usage.usage_lib import UsageContext
25
+
26
+CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
27
+ SchedulerConfig, LoRAConfig]
28
+
29
+logger = init_logger(__name__)
30
+
31
+POLLING_TIMEOUT_MS = 10000
32
+HEALTHY_RESPONSE = (pickle.dumps(VLLM_RPC_SUCCESS_STR), )
33
+
34
+
35
+class MQLLMEngine:
36
+ """A multiprocessing wrapper for :class:`LLMEngine`.
37
+
38
+ This class is used to wrap the :class:`LLMEngine` class to enable use
39
+ in concurrnet manner. It runs a background loop and uses zeromq to
40
+ receive new requests and stream outputs incrementally via ipc.
41
+
42
+ The :class:`LLMEngine.generate` is kicked off when a new
43
+ RPCGenerateRequest is received by the input_socket.
44
+
45
+ The self.engine_loop checks the input_socket for new requests,
46
+ adds them to the LLMEngine if there are any, calls the internal
47
+ :class:`LLMEngine.step()`, and sends the RequestOutputs back over
48
+ the output_socket.
49
+
50
+ If use_async_sockets is set, the logic associated with reading new
51
+ requests from the socket and sending data to the socket is passed
52
+ as a callback to the llm_engine, which calls the logic asynchronously
53
+ such that the IPC can be overlapped with the GPU.
54
+
55
+ Args:
56
+ ipc_path: Base path for zeromq interprocess messaging
57
+ use_async_sockets: Whether to make send/recv async with GPU
58
+ log_requests: Whether to log the requests.
59
+ *args: Arguments for :class:`LLMEngine`.
60
+ **kwargs: Arguments for :class:`LLMEngine`.
61
+ """
62
+
63
+ def __init__(self,
64
+ ipc_path: str,
65
+ use_async_sockets: bool,
66
+ *args,
67
+ log_requests: bool = True,
68
+ **kwargs) -> None:
69
+ self.engine = LLMEngine(*args, **kwargs)
70
+ self.log_requests = log_requests
71
+
72
+ self.use_async_sockets = use_async_sockets
73
+ if self.use_async_sockets:
74
+ self.engine.process_request_outputs_callback = \
75
+ self._async_socket_engine_callback
76
+
77
+ self.ctx = zmq.Context() # type: ignore[attr-defined]
78
+
79
+ # Receive input from the client.
80
+ self.input_socket = self.ctx.socket(zmq.constants.PULL)
81
+ self.input_socket.bind(f"{ipc_path}{IPC_INPUT_EXT}")
82
+
83
+ # Send output stream back to client.
84
+ self.output_socket = self.ctx.socket(zmq.constants.PUSH)
85
+ self.output_socket.bind(f"{ipc_path}{IPC_OUTPUT_EXT}")
86
+
87
+ # Send health status back to client.
88
+ self.health_socket = self.ctx.socket(zmq.constants.PUSH)
89
+ self.health_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}")
90
+
91
+ # IPC path for the data socket.
92
+ self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
93
+
94
+ # Error state.
95
+ self._errored_with: Optional[BaseException] = None
96
+
97
+ @property
98
+ def dead_error(self) -> BaseException:
99
+ if self._errored_with is not None:
100
+ return ENGINE_DEAD_ERROR(self._errored_with)
101
+ else:
102
+ return ENGINE_DEAD_ERROR()
103
+
104
+ @classmethod
105
+ def from_engine_args(cls, engine_args: AsyncEngineArgs,
106
+ usage_context: UsageContext, ipc_path: str):
107
+ """Creates an MQLLMEngine from the engine arguments."""
108
+
109
+ engine_config = engine_args.create_engine_config()
110
+
111
+ executor_class = LLMEngine._get_executor_cls(engine_config)
112
+
113
+ return cls(
114
+ ipc_path=ipc_path,
115
+ use_async_sockets=engine_config.model_config.use_async_output_proc,
116
+ **engine_config.to_dict(),
117
+ executor_class=executor_class,
118
+ log_requests=not engine_args.disable_log_requests,
119
+ log_stats=not engine_args.disable_log_stats,
120
+ usage_context=usage_context)
121
+
122
+ def start(self):
123
+ try:
124
+ try:
125
+ logger.debug("Starting Startup Loop.")
126
+ self.run_startup_loop()
127
+ logger.debug("Starting Engine Loop.")
128
+ self.run_engine_loop()
129
+ except Exception as e:
130
+ logger.exception(repr(e))
131
+ except KeyboardInterrupt:
132
+ logger.debug("Shutting down MQLLMEngine.")
133
+ finally:
134
+ logger.debug("MQLLMEngine is shut down.")
135
+ self.cleanup()
136
+
137
+ def cleanup(self):
138
+ """Cleanup zeromq state on shutdown."""
139
+ # Closes all sockets and destroys context.
140
+ self.ctx.destroy(linger=0)
141
+ del self.engine
142
+
143
+ @contextmanager
144
+ def make_data_socket(
145
+ self) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
146
+ socket = self.ctx.socket(zmq.constants.ROUTER)
147
+ try:
148
+ socket.bind(self.data_ipc_path)
149
+ yield socket
150
+ finally:
151
+ socket.close(linger=0)
152
+
153
+ def run_startup_loop(self) -> None:
154
+ """Startup loop for sending data from Engine -> Client."""
155
+
156
+ with self.make_data_socket() as socket:
157
+ response: Union[RPCStartupResponse, BaseException]
158
+ try:
159
+ identity, message = socket.recv_multipart(copy=False)
160
+ request: RPCStartupRequest = pickle.loads(message.buffer)
161
+
162
+ # Handle the query from the Client.
163
+ if request == RPCStartupRequest.IS_SERVER_READY:
164
+ tracing_enabled = self.engine.is_tracing_enabled()
165
+ response = RPCStartupResponse(
166
+ tracing_enabled=tracing_enabled)
167
+
168
+ except Exception as e:
169
+ response = e
170
+
171
+ socket.send_multipart((identity, pickle.dumps(response)),
172
+ copy=False)
173
+
174
+ def run_engine_loop(self):
175
+ """Core busy loop of the LLMEngine."""
176
+
177
+ while True:
178
+ if not self.engine.has_unfinished_requests():
179
+ # Poll until there is work to do.
180
+ while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
181
+ self.engine.do_log_stats()
182
+ logger.debug("Waiting for new requests in engine loop.")
183
+
184
+ # Handle any input from the client.
185
+ self.handle_new_input()
186
+
187
+ # Engine step.
188
+ request_outputs = self.engine_step()
189
+
190
+ # Send request outputs (if async, done in engine_step callback).
191
+ if not self.use_async_sockets:
192
+ self._send_outputs(request_outputs)
193
+
194
+ def engine_step(self) -> List[RequestOutput]:
195
+ """Engine step wrapper with error handling."""
196
+
197
+ try:
198
+ return self.engine.step()
199
+ except SystemExit:
200
+ raise
201
+ except BaseException as e:
202
+ self._set_errored(e)
203
+ rpc_err = RPCError(request_id=None,
204
+ is_engine_errored=True,
205
+ exception=e)
206
+ self._send_outputs(rpc_err)
207
+ raise e
208
+
209
+ def handle_new_input(self):
210
+ """Handle new input from the socket"""
211
+ try:
212
+ while self.input_socket.poll(timeout=0) != 0:
213
+ frames = self.input_socket.recv_multipart(copy=False)
214
+ request = pickle.loads(frames[0].buffer)
215
+
216
+ if isinstance(request, RPCGenerateRequest):
217
+ if len(frames) > 1:
218
+ # Use cloudpickle for logits processors
219
+ lprocs = cloudpickle.loads(frames[1].buffer)
220
+ request.sampling_params.logits_processors = lprocs
221
+ self._handle_generate_request(request)
222
+ elif isinstance(request, RPCAbortRequest):
223
+ self._handle_abort_request(request)
224
+ elif isinstance(request, RPCHealthRequest):
225
+ self._handle_health_request()
226
+ else:
227
+ raise ValueError("Unknown RPCRequest Type: {request}")
228
+
229
+ except Exception as e:
230
+ self._set_errored(e)
231
+ self._send_unhealthy(e)
232
+ raise e
233
+
234
+ def _handle_generate_request(self, request: RPCGenerateRequest):
235
+ """Handle RPCGenerateRequest by adding it to the LLMEngine."""
236
+ request_id = request.request_id
237
+
238
+ if self._errored_with is not None:
239
+ rpc_err = RPCError(request_id=request_id,
240
+ is_engine_errored=True,
241
+ exception=ENGINE_DEAD_ERROR(self._errored_with))
242
+ self._send_outputs(rpc_err)
243
+
244
+ try:
245
+ self.engine.add_request(
246
+ request_id=request_id,
247
+ inputs=request.inputs,
248
+ params=request.sampling_params,
249
+ lora_request=request.lora_request,
250
+ trace_headers=request.trace_headers,
251
+ prompt_adapter_request=request.prompt_adapter_request)
252
+
253
+ if self.log_requests:
254
+ logger.info("Added request %s.", request.request_id)
255
+
256
+ except Exception as e:
257
+ # We do not set self._errored = True here, since the error
258
+ # is due to an issue adding this request to the engine,
259
+ # rather than an issue with the engine itself.
260
+ is_errored = self._errored_with is not None
261
+ rpc_err = RPCError(request_id=request_id,
262
+ is_engine_errored=is_errored,
263
+ exception=e)
264
+ self._send_outputs(rpc_err)
265
+
266
+ # Remove request from the engine.
267
+ self.engine.abort_request(request_id)
268
+
269
+ def _handle_abort_request(self, request: RPCAbortRequest):
270
+ self.engine.abort_request(request.request_id)
271
+ if self.log_requests:
272
+ logger.info("Aborted request %s.", request.request_id)
273
+
274
+ def _handle_health_request(self):
275
+ if self._errored_with is not None:
276
+ self._send_unhealthy(self._errored_with)
277
+
278
+ # Raises error if unhealthy.
279
+ self.engine.check_health()
280
+ self._send_healthy()
281
+
282
+ def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
283
+ """Send List of RequestOutput to RPCClient."""
284
+ if outputs:
285
+ output_bytes = pickle.dumps(outputs)
286
+ self.output_socket.send_multipart((output_bytes, ), copy=False)
287
+
288
+ def _send_healthy(self):
289
+ """Send HEALTHY message to RPCClient."""
290
+ self.health_socket.send_multipart(HEALTHY_RESPONSE, copy=False)
291
+
292
+ def _send_unhealthy(self, error: BaseException):
293
+ """Send UNHEALTHY message to RPCClient."""
294
+ error_bytes = pickle.dumps(error)
295
+ self.health_socket.send_multipart((error_bytes, ), copy=False)
296
+
297
+ def _async_socket_engine_callback(self,
298
+ request_outputs: REQUEST_OUTPUTS_T):
299
+ """Callback used by engine to make socket handling async with GPU."""
300
+ self._send_outputs(request_outputs)
301
+ self.handle_new_input()
302
+
303
+ def _set_errored(self, e: BaseException):
304
+ """Log and set errored status if this is the first issue."""
305
+ if self._errored_with is None:
306
+ self._errored_with = e
307
+
308
+
309
+def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
310
+ ipc_path: str):
311
+
312
+ def signal_handler(*_) -> None:
313
+ # Interrupt server on sigterm
314
+ raise KeyboardInterrupt("MQLLMEngine terminated")
315
+
316
+ signal.signal(signal.SIGTERM, signal_handler)
317
+
318
+ engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
319
+ usage_context=usage_context,
320
+ ipc_path=ipc_path)
321
+ engine.start()
vllm/engine/protocol.py
+4
-4
@@ -14,8 +14,8 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
14
15
16
@runtime_checkable
17
-class AsyncEngineClient(Protocol):
18
- """Protocol class for Clients to AsyncLLMEngine"""
17
+class EngineClient(Protocol):
18
+ """Protocol class for Clients to Engine"""
19
20
@property
21
def is_running(self) -> bool:
@@ -30,8 +30,8 @@ class AsyncEngineClient(Protocol):
30
...
31
32
@property
33
- def limit_concurrency(self) -> Optional[int]:
34
- """Maximum number of concurrently running requests."""
33
+ def dead_error(self) -> BaseException:
34
+ ...
35
36
def generate(
37
self,
vllm/entrypoints/launcher.py
+16
-14
@@ -1,21 +1,21 @@
1
import asyncio
2
import signal
3
from http import HTTPStatus
4
-from typing import Any, Optional
4
+from typing import Any
5
6
import uvicorn
7
from fastapi import FastAPI, Request, Response
8
9
from vllm import envs
10
from vllm.engine.async_llm_engine import AsyncEngineDeadError
11
+from vllm.engine.multiprocessing import MQEngineDeadError
12
from vllm.logger import init_logger
13
from vllm.utils import find_process_using_port
14
15
logger = init_logger(__name__)
16
17
17
-async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
18
- **uvicorn_kwargs: Any):
18
+async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
19
logger.info("Available routes are:")
20
for route in app.routes:
21
methods = getattr(route, "methods", None)
@@ -26,15 +26,6 @@ async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
26
27
logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
28
29
- # Set concurrency limits in uvicorn if running in multiprocessing mode
30
- # since zmq has maximum socket limit of zmq.constants.SOCKET_LIMIT (65536).
31
- if limit_concurrency is not None:
32
- logger.info(
33
- "Launching Uvicorn with --limit_concurrency %s. To avoid this "
34
- "limit at the expense of performance run with "
35
- "--disable-frontend-multiprocessing", limit_concurrency)
36
- uvicorn_kwargs["limit_concurrency"] = limit_concurrency
37
-
29
config = uvicorn.Config(app, **uvicorn_kwargs)
30
server = uvicorn.Server(config)
31
_add_shutdown_handlers(app, server)
@@ -63,7 +54,7 @@ async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
54
logger.debug(
55
"port %s is used by process %s launched with command:\n%s",
56
port, process, " ".join(process.cmdline()))
66
- logger.info("Gracefully stopping http server")
57
+ logger.info("Shutting down FastAPI HTTP server.")
58
return server.shutdown()
59
60
@@ -90,7 +81,7 @@ def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
81
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
82
83
@app.exception_handler(AsyncEngineDeadError)
93
- async def engine_dead_handler(_, __):
84
+ async def async_engine_dead_handler(_, __):
85
"""Kill the server if the async engine is already dead. It will
86
not handle any further requests."""
87
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
@@ -99,3 +90,14 @@ def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
90
server.should_exit = True
91
92
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
93
+
94
+ @app.exception_handler(MQEngineDeadError)
95
+ async def mq_engine_dead_handler(_, __):
96
+ """Kill the server if the mq engine is already dead. It will
97
+ not handle any further requests."""
98
+ if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
99
+ logger.fatal("MQLLMEngine is already dead, terminating server "
100
+ "process")
101
+ server.should_exit = True
102
+
103
+ return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
vllm/entrypoints/openai/api_server.py
+55
-66
@@ -26,7 +26,9 @@ import vllm.envs as envs
26
from vllm.config import ModelConfig
27
from vllm.engine.arg_utils import AsyncEngineArgs
28
from vllm.engine.async_llm_engine import AsyncLLMEngine
29
-from vllm.engine.protocol import AsyncEngineClient
29
+from vllm.engine.multiprocessing.client import MQLLMEngineClient
30
+from vllm.engine.multiprocessing.engine import run_mp_engine
31
+from vllm.engine.protocol import EngineClient
32
from vllm.entrypoints.launcher import serve_http
33
from vllm.entrypoints.logger import RequestLogger
34
from vllm.entrypoints.openai.cli_args import make_arg_parser
@@ -44,8 +46,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
46
TokenizeRequest,
47
TokenizeResponse,
48
UnloadLoraAdapterRequest)
47
-from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
48
-from vllm.entrypoints.openai.rpc.server import run_rpc_server
49
# yapf: enable
50
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
51
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -67,29 +67,16 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
67
_running_tasks: Set[asyncio.Task] = set()
68
69
70
-def model_is_embedding(model_name: str, trust_remote_code: bool,
71
- quantization: Optional[str],
72
- revision: Optional[str]) -> bool:
73
- return ModelConfig(model=model_name,
74
- revision=revision,
75
- tokenizer=model_name,
76
- tokenizer_mode="auto",
77
- trust_remote_code=trust_remote_code,
78
- quantization=quantization,
79
- seed=0,
80
- dtype="auto").embedding_mode
81
-
82
-
70
@asynccontextmanager
71
async def lifespan(app: FastAPI):
72
try:
73
if app.state.log_stats:
87
- async_engine_client = app.state.engine_client
74
+ engine_client: EngineClient = app.state.engine_client
75
76
async def _force_log():
77
while True:
91
- await asyncio.sleep(10)
92
- await async_engine_client.do_log_stats()
78
+ await asyncio.sleep(10.)
79
+ await engine_client.do_log_stats()
80
81
task = asyncio.create_task(_force_log())
82
_running_tasks.add(task)
@@ -108,9 +95,9 @@ async def lifespan(app: FastAPI):
95
96
@asynccontextmanager
97
async def build_async_engine_client(
111
- args: Namespace) -> AsyncIterator[Optional[AsyncEngineClient]]:
98
+ args: Namespace) -> AsyncIterator[Optional[EngineClient]]:
99
113
- # Context manager to handle async_engine_client lifecycle
100
+ # Context manager to handle engine_client lifecycle
101
# Ensures everything is shutdown and cleaned up on error/exit
102
engine_args = AsyncEngineArgs.from_cli_args(args)
103
@@ -123,19 +110,18 @@ async def build_async_engine_client(
110
async def build_async_engine_client_from_engine_args(
111
engine_args: AsyncEngineArgs,
112
disable_frontend_multiprocessing: bool = False,
126
-) -> AsyncIterator[Optional[AsyncEngineClient]]:
113
+) -> AsyncIterator[Optional[EngineClient]]:
114
"""
128
- Create AsyncEngineClient, either:
115
+ Create EngineClient, either:
116
- in-process using the AsyncLLMEngine Directly
117
- multiprocess using AsyncLLMEngine RPC
118
119
Returns the Client or None if the creation failed.
120
"""
121
135
- # If manually triggered or embedding model, use AsyncLLMEngine in process.
136
- # TODO: support embedding model via RPC.
137
- if (model_is_embedding(engine_args.model, engine_args.trust_remote_code,
138
- engine_args.quantization, engine_args.revision)
122
+ # Fall back
123
+ # TODO: fill out feature matrix.
124
+ if (MQLLMEngineClient.is_unsupported_config(engine_args)
125
or disable_frontend_multiprocessing):
126
engine_config = engine_args.create_engine_config()
127
uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
@@ -173,56 +159,60 @@ async def build_async_engine_client_from_engine_args(
159
"and vLLM will properly handle cleanup.")
160
161
# Select random path for IPC.
176
- rpc_path = get_open_zmq_ipc_path()
177
- logger.info("Multiprocessing frontend to use %s for RPC Path.",
178
- rpc_path)
179
-
180
- # Build RPCClient, which conforms to AsyncEngineClient Protocol.
181
- # NOTE: Actually, this is not true yet. We still need to support
182
- # embedding models via RPC (see TODO above)
183
- rpc_client = AsyncEngineRPCClient(rpc_path)
162
+ ipc_path = get_open_zmq_ipc_path()
163
+ logger.info("Multiprocessing frontend to use %s for IPC Path.",
164
+ ipc_path)
165
185
- # Start RPCServer in separate process (holds the AsyncLLMEngine).
186
- context = multiprocessing.get_context("spawn")
166
+ # Start RPCServer in separate process (holds the LLMEngine).
167
# the current process might have CUDA context,
168
# so we need to spawn a new process
189
- rpc_server_process = context.Process(
190
- target=run_rpc_server,
191
- args=(engine_args, UsageContext.OPENAI_API_SERVER, rpc_path))
192
- rpc_server_process.start()
193
- logger.info("Started engine process with PID %d",
194
- rpc_server_process.pid)
169
+ context = multiprocessing.get_context("spawn")
170
+
171
+ engine_process = context.Process(target=run_mp_engine,
172
+ args=(engine_args,
173
+ UsageContext.OPENAI_API_SERVER,
174
+ ipc_path))
175
+ engine_process.start()
176
+ logger.info("Started engine process with PID %d", engine_process.pid)
177
+
178
+ # Build RPCClient, which conforms to EngineClient Protocol.
179
+ # NOTE: Actually, this is not true yet. We still need to support
180
+ # embedding models via RPC (see TODO above)
181
+ engine_config = engine_args.create_engine_config()
182
+ mp_engine_client = MQLLMEngineClient(ipc_path, engine_config)
183
184
try:
185
while True:
186
try:
199
- await rpc_client.setup()
187
+ await mp_engine_client.setup()
188
break
189
except TimeoutError:
202
- if not rpc_server_process.is_alive():
203
- logger.error(
204
- "RPCServer process died before responding "
205
- "to readiness probe")
190
+ if not engine_process.is_alive():
191
+ logger.error("Engine process died before responding "
192
+ "to readiness probe")
193
yield None
194
return
195
209
- yield rpc_client # type: ignore[misc]
196
+ yield mp_engine_client # type: ignore[misc]
197
finally:
198
# Ensure rpc server process was terminated
212
- rpc_server_process.terminate()
199
+ engine_process.terminate()
200
201
# Close all open connections to the backend
215
- rpc_client.close()
202
+ mp_engine_client.close()
203
217
- # Wait for server process to join
218
- rpc_server_process.join()
204
+ # Wait for engine process to join
205
+ engine_process.join(4)
206
+ if engine_process.exitcode is None:
207
+ # Kill if taking longer than 5 seconds to stop
208
+ engine_process.kill()
209
210
# Lazy import for prometheus multiprocessing.
211
# We need to set PROMETHEUS_MULTIPROC_DIR environment variable
212
# before prometheus_client is imported.
213
# See https://prometheus.github.io/client_python/multiprocess/
214
from prometheus_client import multiprocess
225
- multiprocess.mark_process_dead(rpc_server_process.pid)
215
+ multiprocess.mark_process_dead(engine_process.pid)
216
217
218
router = APIRouter()
@@ -270,7 +260,7 @@ def embedding(request: Request) -> OpenAIServingEmbedding:
260
return request.app.state.openai_serving_embedding
261
262
273
-def engine_client(request: Request) -> AsyncEngineClient:
263
+def engine_client(request: Request) -> EngineClient:
264
return request.app.state.engine_client
265
266
@@ -473,7 +463,7 @@ def build_app(args: Namespace) -> FastAPI:
463
464
465
def init_app_state(
476
- async_engine_client: AsyncEngineClient,
466
+ engine_client: EngineClient,
467
model_config: ModelConfig,
468
state: State,
469
args: Namespace,
@@ -488,11 +478,11 @@ def init_app_state(
478
else:
479
request_logger = RequestLogger(max_log_len=args.max_log_len)
480
491
- state.engine_client = async_engine_client
481
+ state.engine_client = engine_client
482
state.log_stats = not args.disable_log_stats
483
484
state.openai_serving_chat = OpenAIServingChat(
495
- async_engine_client,
485
+ engine_client,
486
model_config,
487
served_model_names,
488
args.response_role,
@@ -504,7 +494,7 @@ def init_app_state(
494
enable_auto_tools=args.enable_auto_tool_choice,
495
tool_parser=args.tool_call_parser)
496
state.openai_serving_completion = OpenAIServingCompletion(
507
- async_engine_client,
497
+ engine_client,
498
model_config,
499
served_model_names,
500
lora_modules=args.lora_modules,
@@ -513,13 +503,13 @@ def init_app_state(
503
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
504
)
505
state.openai_serving_embedding = OpenAIServingEmbedding(
516
- async_engine_client,
506
+ engine_client,
507
model_config,
508
served_model_names,
509
request_logger=request_logger,
510
)
511
state.openai_serving_tokenization = OpenAIServingTokenization(
522
- async_engine_client,
512
+ engine_client,
513
model_config,
514
served_model_names,
515
lora_modules=args.lora_modules,
@@ -541,21 +531,20 @@ async def run_server(args, **uvicorn_kwargs) -> None:
531
532
signal.signal(signal.SIGTERM, signal_handler)
533
544
- async with build_async_engine_client(args) as async_engine_client:
534
+ async with build_async_engine_client(args) as engine_client:
535
# If None, creation of the client failed and we exit.
546
- if async_engine_client is None:
536
+ if engine_client is None:
537
return
538
539
app = build_app(args)
540
551
- model_config = await async_engine_client.get_model_config()
552
- init_app_state(async_engine_client, model_config, app.state, args)
541
+ model_config = await engine_client.get_model_config()
542
+ init_app_state(engine_client, model_config, app.state, args)
543
544
temp_socket.close()
545
546
shutdown_task = await serve_http(
547
app,
558
- limit_concurrency=async_engine_client.limit_concurrency,
548
host=args.host,
549
port=args.port,
550
log_level=args.uvicorn_log_level,
vllm/entrypoints/openai/rpc/__init__.py
deleted
-50
@@ -1,50 +0,0 @@
1
-from dataclasses import dataclass
2
-from enum import Enum
3
-from typing import Mapping, Optional, Union
4
-
5
-from vllm.inputs import PromptInputs
6
-from vllm.lora.request import LoRARequest
7
-from vllm.prompt_adapter.request import PromptAdapterRequest
8
-from vllm.sampling_params import SamplingParams
9
-
10
-# Success string used for RPC instructions.
11
-VLLM_RPC_SUCCESS_STR = "SUCCESS"
12
-
13
-# Minimum value of ZMQ.SOCKET_LIMIT to run mp.
14
-VLLM_RPC_SOCKET_LIMIT_CUTOFF = 2000
15
-
16
-# HWM is set to Infinity.
17
-VLLM_RPC_ZMQ_HWM = 0
18
-
19
-
20
-@dataclass
21
-class RPCGenerateRequest:
22
- inputs: PromptInputs
23
- sampling_params: SamplingParams
24
- request_id: str
25
- lora_request: Optional[LoRARequest] = None
26
- trace_headers: Optional[Mapping[str, str]] = None
27
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
28
-
29
-
30
-@dataclass
31
-class RPCAbortRequest:
32
- request_id: str
33
-
34
-
35
-class RPCUtilityRequest(Enum):
36
- IS_SERVER_READY = 1
37
- GET_MODEL_CONFIG = 2
38
- GET_DECODING_CONFIG = 3
39
- GET_PARALLEL_CONFIG = 4
40
- GET_SCHEDULER_CONFIG = 5
41
- GET_LORA_CONFIG = 6
42
- DO_LOG_STATS = 7
43
- IS_SERVER_HEALTHY = 8
44
- IS_TRACING_ENABLED = 9
45
- START_PROFILE = 10
46
- STOP_PROFILE = 11
47
-
48
-
49
-RPC_REQUEST_TYPE = Union[RPCGenerateRequest, RPCAbortRequest,
50
- RPCUtilityRequest]
vllm/entrypoints/openai/rpc/client.py
deleted
-451
@@ -1,451 +0,0 @@
1
-import asyncio
2
-import pickle
3
-from contextlib import contextmanager, suppress
4
-from typing import Any, AsyncGenerator, Iterator, Mapping, Optional
5
-from uuid import uuid4
6
-
7
-import cloudpickle
8
-import zmq
9
-import zmq.asyncio
10
-from zmq import Frame # type: ignore[attr-defined]
11
-from zmq.asyncio import Socket
12
-
13
-from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
14
- ParallelConfig, SchedulerConfig)
15
-# yapf: disable
16
-from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
17
- VLLM_RPC_SOCKET_LIMIT_CUTOFF,
18
- VLLM_RPC_SUCCESS_STR,
19
- VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
20
- RPCGenerateRequest, RPCUtilityRequest)
21
-# yapf: enable
22
-from vllm.envs import VLLM_RPC_GET_DATA_TIMEOUT_MS
23
-from vllm.inputs import PromptInputs
24
-from vllm.logger import init_logger
25
-from vllm.lora.request import LoRARequest
26
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27
-from vllm.prompt_adapter.request import PromptAdapterRequest
28
-from vllm.sampling_params import SamplingParams
29
-from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
30
-
31
-logger = init_logger(__name__)
32
-
33
-# Path used for inprocess proxy.
34
-INPROC_PROXY_PATH = f"inproc://{uuid4()}"
35
-
36
-
37
-class RPCClientClosedError(Exception):
38
- """Exception class raised when the client is used post-close.
39
-
40
- The client can be closed, which closes the ZMQ context. This normally
41
- happens on server shutdown. In some cases, methods like abort and
42
- do_log_stats will still be called and then try to open a socket, which
43
- causes a ZMQError and creates a huge stack trace.
44
- So, we throw this error such that we can suppress it.
45
- """
46
-
47
-
48
-class AsyncEngineRPCClient:
49
- """
50
- RPCClient that connects to the RPCServer wrapping AsyncLLMEngine.
51
-
52
- The overall design mirrors the Asynchronous Client Server Pattern
53
- https://zguide.zeromq.org/docs/chapter3/#The-Asynchronous-Client-Server-Pattern
54
-
55
- On startup, the RPCClient:
56
- - makes DEALER socket (to_rpc_server) that connects to the RPCServer
57
- via ipc, which uses unix sockets under the hood
58
- (https://libzmq.readthedocs.io/en/zeromq4-1/zmq_ipc.html)
59
- - makes ROUTER socket (from_api_server) that binds to a random
60
- inproc address, which uses memory under the hood
61
- (https://libzmq.readthedocs.io/en/zeromq3-x/zmq_inproc.html)
62
- - runs a proxy in a background asyncio task between
63
- from_api_server (ROUTER, inproc) and to_rpc_server (DEALER ipc, )
64
-
65
- Each request handled by the asyncio api_server calls generate():
66
- - make a DEALER socket that connects to from_api_server via inproc
67
- - send a RCPGenerateRequest to the inproc socket
68
- - background proxy forwards the request from inproc -> ipc
69
- - RPCServer responds to the request one token at a time over ipc
70
- - background proxy forwards the response from ipc -> inproc
71
-
72
- The connection looks like this:
73
- DEALER <- inproc -> [ ROUTER | DEALER ] <- ipc -> DEALER
74
-
75
- Message routing is performed via identities that are managed by the
76
- ROUTER socket. ROUTER sockets track every connection it has and
77
- tells the caller about these. The way it tells the caller is to stick
78
- the connection identity in front of each message received. When we
79
- send the message via a ROUTER, we first send an identity frame.
80
- See https://zguide.zeromq.org/docs/chapter3/#The-Extended-Reply-Envelope
81
- for more details on connection identities.
82
-
83
- This proxy design enables us to use a single unix socket, which
84
- improves performance by avoiding syscalls (~5%) and avoids resource limits
85
- such as ulimit, which defaults to 1024 on ubuntu.
86
-
87
- Note: we run set_hwm(0) on each socket, which sets the HWM to inf,
88
- which is required to avoid dropping messages under high load.
89
- This is generally not advisable. However, since we are in control
90
- of both sides of the connection + failure on either side is
91
- catastrophic to the overall system health and memory profiling
92
- suggests limited memory overhead relative to asyncio, we will
93
- proceed for now.
94
-
95
- See https://zguide.zeromq.org/docs/chapter2/#High-Water-Marks
96
- for more details on high water marks.
97
- """
98
-
99
- def __init__(self, rpc_path: str):
100
- self.context = zmq.asyncio.Context()
101
- self._data_timeout = VLLM_RPC_GET_DATA_TIMEOUT_MS
102
- self._errored = False
103
-
104
- # Maximum number of sockets that can be opened (typically 65536).
105
- # ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
106
- socket_limit = self.context.get(zmq.constants.SOCKET_LIMIT)
107
- assert isinstance(socket_limit, int)
108
- if socket_limit < VLLM_RPC_SOCKET_LIMIT_CUTOFF:
109
- raise ValueError(
110
- f"Found zmq.constants.SOCKET_LIMIT={socket_limit}, which caps "
111
- "the number of concurrent requests vLLM can process. Launch "
112
- "vLLM with --disable-frontend-multiprocessing and open a "
113
- "GitHub issue so we can investigate.")
114
-
115
- # We only have 1 ipc connection that uses unix sockets, so
116
- # safe to set MAX_SOCKETS to the zmq SOCKET_LIMIT (i.e. will
117
- # not run into ulimit issues)
118
- self.context.set(zmq.constants.MAX_SOCKETS, socket_limit)
119
-
120
- # IPC connection to RPC Server (uses unix sockets).
121
- self.to_rpc_server: Socket = self.context.socket(zmq.constants.DEALER)
122
- self.to_rpc_server.set_hwm(VLLM_RPC_ZMQ_HWM)
123
- self.to_rpc_server.bind(rpc_path)
124
-
125
- # In process proxy to RPC Server (uses memory-based messaging).
126
- self.from_api_server: Socket = self.context.socket(
127
- zmq.constants.ROUTER)
128
- self.from_api_server.set_hwm(VLLM_RPC_ZMQ_HWM)
129
- self.from_api_server.bind(INPROC_PROXY_PATH)
130
-
131
- # Asyncio background task for the proxy.
132
- self.proxy_in_task = asyncio.create_task(
133
- self.run_proxy(self.from_api_server, self.to_rpc_server))
134
- self.proxy_out_task = asyncio.create_task(
135
- self.run_proxy(self.to_rpc_server, self.from_api_server))
136
-
137
- # Since we open 1 inproc socket per request, we have a hard cap on
138
- # the number of requests that can run in vLLM w. frontend
139
- # mulitprocessing. This value is used uvicorn to launch
140
- # with --limit-concurrency to return 503 when server is overloaded.
141
- # We need 2 sockets per request - 2:
142
- # 1 for generate(), 1 for abort(), do_log_stats(), check_health()
143
- self.limit_concurrency = socket_limit // 2 - 2
144
-
145
- async def run_proxy(self, socket_from: Socket, socket_to: Socket):
146
- """Background task that runs a proxy"""
147
- while True:
148
- frames = await socket_from.recv_multipart(copy=False)
149
- await socket_to.send_multipart(frames, copy=False)
150
-
151
- async def setup(self):
152
- """Setup the client before it starts sending server requests."""
153
-
154
- # Wait until server is ready.
155
- await self._wait_for_server_rpc()
156
-
157
- # Get the configs.
158
- self.model_config = await self._get_model_config_rpc()
159
- self.decoding_config = await self._get_decoding_config_rpc()
160
- self.tracing_flag = await self._is_tracing_enabled_rpc()
161
-
162
- # Create the tokenizer group.
163
- # TODO: refactor OAI server to avoid needing this info.
164
- self.tokenizer = init_tokenizer_from_configs(
165
- model_config=self.model_config,
166
- scheduler_config=(await self._get_scheduler_config_rpc()),
167
- parallel_config=(await self._get_parallel_config_rpc()),
168
- enable_lora=bool(await self._get_lora_config_rpc()),
169
- )
170
-
171
- def close(self):
172
- """Destroy the ZeroMQ Context."""
173
- # Close all sockets associated with this context and
174
- # then terminate the context.
175
- self.from_api_server.close()
176
- self.to_rpc_server.close()
177
- self.context.destroy()
178
-
179
- @contextmanager
180
- def to_proxy_socket(self) -> Iterator[Socket]:
181
- # Connect to the RPCServer via the proxy.
182
-
183
- # Raise a sensible error if the client was already closed.
184
- # This can happen if a server shutdown is triggered but some coroutines
185
- # are still running requests.
186
- # There should not be a race condition with this check because we don't
187
- # yield to the event loop between here and opening the socket.
188
- if self.context.closed:
189
- raise RPCClientClosedError("The ZMQ client has already shut down")
190
-
191
- # Note that we use DEALER to enable asynchronous communication
192
- # to enable streaming.
193
- socket = self.context.socket(zmq.constants.DEALER)
194
- socket.set_hwm(VLLM_RPC_ZMQ_HWM)
195
- try:
196
- socket.connect(INPROC_PROXY_PATH)
197
- yield socket
198
- finally:
199
- socket.close(linger=0)
200
-
201
- async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
202
- expected_type: Any,
203
- error_message: str) -> Any:
204
- """Send an RPC request that is expecting data back."""
205
-
206
- with self.to_proxy_socket() as socket:
207
- # Ping RPCServer with a request.
208
- await socket.send_multipart((cloudpickle.dumps(request), ),
209
- copy=False)
210
-
211
- # Make sure the server responds
212
- if await socket.poll(timeout=self._data_timeout) == 0:
213
- raise TimeoutError("Server didn't reply within "
214
- f"{self._data_timeout} ms")
215
-
216
- # Await the data from the Server.
217
- frame = await socket.recv(copy=False)
218
- assert isinstance(frame, Frame)
219
- data = pickle.loads(frame.buffer)
220
-
221
- if isinstance(data, Exception):
222
- # Re-raise exceptions returned by the server
223
- raise data
224
-
225
- if not isinstance(data, expected_type):
226
- # LoRAConfig can be None.
227
- if expected_type == LoRAConfig and data is None:
228
- pass
229
- elif isinstance(data, Exception):
230
- logger.error(error_message)
231
- raise data
232
- else:
233
- raise ValueError(error_message)
234
-
235
- return data
236
-
237
- async def _send_one_way_rpc_request(self,
238
- request: RPC_REQUEST_TYPE,
239
- error_message: str,
240
- socket: Optional[Socket] = None):
241
- """Send one-way RPC request to trigger an action."""
242
-
243
- async def do_rpc_call(socket: Socket, request: RPC_REQUEST_TYPE):
244
-
245
- await socket.send_multipart((cloudpickle.dumps(request), ))
246
-
247
- if await socket.poll(timeout=self._data_timeout) == 0:
248
- raise TimeoutError("Server didn't reply within "
249
- f"{self._data_timeout} ms")
250
-
251
- frame = await socket.recv(copy=False)
252
- assert isinstance(frame, Frame)
253
- return pickle.loads(frame.buffer)
254
-
255
- # Make a new socket connection.
256
- if socket is None:
257
- with self.to_proxy_socket() as socket:
258
- response = await do_rpc_call(socket, request)
259
-
260
- # Use existing socket connection.
261
- else:
262
- response = await do_rpc_call(socket, request)
263
-
264
- if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
265
- if isinstance(response, Exception):
266
- logger.error(error_message)
267
- raise response
268
- raise ValueError(error_message)
269
-
270
- async def get_tokenizer(self, lora_request: LoRARequest):
271
- return await self.tokenizer.get_lora_tokenizer_async(lora_request)
272
-
273
- async def get_decoding_config(self) -> DecodingConfig:
274
- return self.decoding_config
275
-
276
- async def get_model_config(self) -> ModelConfig:
277
- return self.model_config
278
-
279
- async def is_tracing_enabled(self) -> bool:
280
- return self.tracing_flag
281
-
282
- async def _wait_for_server_rpc(self):
283
- """Wait for the RPCServer to start up."""
284
-
285
- await self._send_one_way_rpc_request(
286
- request=RPCUtilityRequest.IS_SERVER_READY,
287
- error_message="Unable to start RPC Server")
288
-
289
- async def _get_model_config_rpc(self) -> ModelConfig:
290
- """Get the ModelConfig object from the RPC Server"""
291
-
292
- return await self._send_get_data_rpc_request(
293
- RPCUtilityRequest.GET_MODEL_CONFIG,
294
- expected_type=ModelConfig,
295
- error_message="Could not get ModelConfig from RPC Server")
296
-
297
- async def _get_decoding_config_rpc(self) -> DecodingConfig:
298
- """Get DecodingConfig from the RPCServer"""
299
-
300
- return await self._send_get_data_rpc_request(
301
- RPCUtilityRequest.GET_DECODING_CONFIG,
302
- expected_type=DecodingConfig,
303
- error_message="Could not get DecodingConfig from RPC Server")
304
-
305
- async def _get_parallel_config_rpc(self) -> ParallelConfig:
306
- """Get ParallelConfig from the RPCServer"""
307
-
308
- return await self._send_get_data_rpc_request(
309
- RPCUtilityRequest.GET_PARALLEL_CONFIG,
310
- expected_type=ParallelConfig,
311
- error_message="Could not get ParallelConfig from RPC Server")
312
-
313
- async def _get_scheduler_config_rpc(self) -> SchedulerConfig:
314
- """Get SchedulerConfig from the RPCServer"""
315
-
316
- return await self._send_get_data_rpc_request(
317
- RPCUtilityRequest.GET_SCHEDULER_CONFIG,
318
- expected_type=SchedulerConfig,
319
- error_message="Could not get SchedulerConfig from RPC Server")
320
-
321
- async def _get_lora_config_rpc(self) -> LoRAConfig:
322
- """Get LoRAConfig from the RPCServer"""
323
-
324
- return await self._send_get_data_rpc_request(
325
- RPCUtilityRequest.GET_LORA_CONFIG,
326
- expected_type=LoRAConfig,
327
- error_message="Could not get LoRAConfig from RPC Server")
328
-
329
- async def _is_tracing_enabled_rpc(self) -> bool:
330
- """Get is_tracing_enabled flag from the RPCServer"""
331
-
332
- return await self._send_get_data_rpc_request(
333
- RPCUtilityRequest.IS_TRACING_ENABLED,
334
- expected_type=bool,
335
- error_message="Could not get is_tracing_enabled from RPC Server")
336
-
337
- async def abort(self, request_id: str):
338
- """Send an ABORT_REQUEST signal to the RPC Server"""
339
-
340
- # Suppress timeouts as well.
341
- # In cases where the server is busy processing requests and a very
342
- # large volume of abort requests arrive, it is likely that the server
343
- # will not be able to ack all of them in time. We have seen this when
344
- # we abort 20k requests at once while another 2k are processing- many
345
- # of them time out, but we see the server successfully abort all of the
346
- # requests.
347
- # In this case we assume that the server has received or will receive
348
- # these abort requests, and ignore the timeout. This prevents a massive
349
- # wall of `TimeoutError` stack traces.
350
- with suppress(RPCClientClosedError, TimeoutError):
351
- await self._send_one_way_rpc_request(
352
- request=RPCAbortRequest(request_id),
353
- error_message=f"RPCAbortRequest {request_id} failed")
354
-
355
- async def do_log_stats(self):
356
- """Send a DO_LOG_STATS signal to the RPC Server"""
357
- with suppress(RPCClientClosedError):
358
- await self._send_one_way_rpc_request(
359
- request=RPCUtilityRequest.DO_LOG_STATS,
360
- error_message="RPCRequest DO_LOG_STATS failed.")
361
-
362
- @property
363
- def is_running(self) -> bool:
364
- return not self._errored
365
-
366
- @property
367
- def is_stopped(self) -> bool:
368
- return self._errored
369
-
370
- @property
371
- def errored(self) -> bool:
372
- return self._errored
373
-
374
- async def generate(
375
- self,
376
- inputs: PromptInputs,
377
- sampling_params: SamplingParams,
378
- request_id: str,
379
- lora_request: Optional[LoRARequest] = None,
380
- trace_headers: Optional[Mapping[str, str]] = None,
381
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
382
- ) -> AsyncGenerator[RequestOutput, None]:
383
- """Send an RPCGenerateRequest to the RPCServer and stream responses."""
384
-
385
- finished = False
386
- try:
387
- with self.to_proxy_socket() as socket:
388
- # Send RPCGenerateRequest to the RPCServer.
389
- await socket.send_multipart((cloudpickle.dumps(
390
- RPCGenerateRequest(
391
- inputs=inputs,
392
- sampling_params=sampling_params,
393
- request_id=request_id,
394
- lora_request=lora_request,
395
- trace_headers=trace_headers,
396
- prompt_adapter_request=prompt_adapter_request)), ))
397
-
398
- # Stream back the results from the RPC Server.
399
- while not finished:
400
- message = await socket.recv(copy=False)
401
- assert isinstance(message, Frame)
402
- request_output = pickle.loads(message.buffer)
403
-
404
- if isinstance(request_output, Exception):
405
- # On exception, check if the server is still healthy
406
- # possibly setting the `errored` property.
407
- if not self._errored:
408
- try:
409
- await self.check_health(socket=socket)
410
- except Exception as e:
411
- self._errored = True
412
- logger.exception(repr(e))
413
-
414
- # NB: do before raising here so that the flag is set
415
- # by the time the caller receives this exception
416
- raise request_output
417
-
418
- finished = request_output.finished
419
- yield request_output
420
-
421
- finally:
422
- # Request was canceled by the client.
423
- if not finished and not self._errored:
424
- await self.abort(request_id)
425
-
426
- async def check_health(self, socket: Optional[Socket] = None) -> None:
427
- """Raise if unhealthy"""
428
-
429
- await self._send_one_way_rpc_request(
430
- request=RPCUtilityRequest.IS_SERVER_HEALTHY,
431
- error_message="Got Unhealthy response from RPC Server",
432
- socket=socket)
433
-
434
- async def encode(self, *args,
435
- **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
436
- raise NotImplementedError(
437
- "Embeddings not supported with multiprocessing backend")
438
-
439
- async def start_profile(self) -> None:
440
- """Start profiling the engine"""
441
-
442
- await self._send_one_way_rpc_request(
443
- request=RPCUtilityRequest.START_PROFILE,
444
- error_message="RPCRequest START_PROFILE failed.")
445
-
446
- async def stop_profile(self) -> None:
447
- """Stop profiling the engine"""
448
-
449
- await self._send_one_way_rpc_request(
450
- request=RPCUtilityRequest.STOP_PROFILE,
451
- error_message="RPCRequest STOP_PROFILE failed.")
vllm/entrypoints/openai/rpc/server.py
deleted
-243
@@ -1,243 +0,0 @@
1
-import asyncio
2
-import pickle
3
-import signal
4
-from typing import Any, Coroutine, Union
5
-
6
-import cloudpickle
7
-import uvloop
8
-import zmq
9
-import zmq.asyncio
10
-from typing_extensions import Never
11
-from zmq import Frame # type: ignore[attr-defined]
12
-from zmq.asyncio import Socket
13
-
14
-from vllm import AsyncEngineArgs, AsyncLLMEngine
15
-from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
16
- ParallelConfig, SchedulerConfig)
17
-from vllm.entrypoints.openai.rpc import (VLLM_RPC_SUCCESS_STR,
18
- VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
19
- RPCGenerateRequest, RPCUtilityRequest)
20
-from vllm.logger import init_logger
21
-from vllm.usage.usage_lib import UsageContext
22
-
23
-logger = init_logger(__name__)
24
-
25
-CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
26
- SchedulerConfig, LoRAConfig]
27
-
28
-
29
-class AsyncEngineRPCServer:
30
-
31
- def __init__(self, async_engine_args: AsyncEngineArgs,
32
- usage_context: UsageContext, rpc_path: str):
33
- # Initialize engine first.
34
- self.engine = AsyncLLMEngine.from_engine_args(
35
- async_engine_args, usage_context=usage_context)
36
-
37
- # Initialize context.
38
- self.context = zmq.asyncio.Context()
39
-
40
- # Init socket.
41
- self.socket: Socket = self.context.socket(zmq.constants.DEALER)
42
- self.socket.set_hwm(VLLM_RPC_ZMQ_HWM)
43
- self.socket.connect(rpc_path)
44
-
45
- def cleanup(self):
46
- """Cleanup all resources."""
47
- self.socket.close()
48
- self.context.destroy()
49
- # Clear the engine reference so that it can be GC'ed.
50
- del self.engine
51
-
52
- async def get_config(self, identity, request):
53
- try:
54
- config: CONFIG_TYPE
55
- if request == RPCUtilityRequest.GET_MODEL_CONFIG:
56
- config = await self.engine.get_model_config()
57
- elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
58
- config = await self.engine.get_decoding_config()
59
- elif request == RPCUtilityRequest.GET_LORA_CONFIG:
60
- config = await self.engine.get_lora_config()
61
- elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
62
- config = await self.engine.get_scheduler_config()
63
- elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
64
- config = await self.engine.get_parallel_config()
65
- else:
66
- raise ValueError("Unknown Config Request: %s", request)
67
-
68
- await self.socket.send_multipart((identity, pickle.dumps(config)),
69
- copy=False)
70
-
71
- except Exception as e:
72
- await self.socket.send_multipart((identity, pickle.dumps(e)),
73
- copy=False)
74
-
75
- async def is_tracing_enabled(self, identity):
76
- """Send the is_tracing_enabled flag"""
77
- tracing_flag = await self.engine.is_tracing_enabled()
78
-
79
- await self.socket.send_multipart(
80
- (identity, pickle.dumps(tracing_flag)))
81
-
82
- async def do_log_stats(self, identity):
83
- """Log stats and confirm success."""
84
- await self.engine.do_log_stats()
85
-
86
- await self.socket.send_multipart(
87
- (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
88
-
89
- async def is_server_ready(self, identity):
90
- """Notify the client that we are ready."""
91
- await self.socket.send_multipart(
92
- (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
93
-
94
- async def abort(self, identity, request: RPCAbortRequest):
95
- """Abort request and notify the client of success."""
96
- try:
97
- # Abort the request in the llm engine.
98
- await self.engine.abort(request.request_id)
99
- result: Union[str, Exception] = VLLM_RPC_SUCCESS_STR
100
- except Exception as e:
101
- result = e
102
- await self.socket.send_multipart((identity, pickle.dumps(result)))
103
-
104
- async def generate(self, identity, generate_request: RPCGenerateRequest):
105
- try:
106
- results_generator = self.engine.generate(
107
- generate_request.inputs,
108
- sampling_params=generate_request.sampling_params,
109
- request_id=generate_request.request_id,
110
- lora_request=generate_request.lora_request,
111
- trace_headers=generate_request.trace_headers,
112
- prompt_adapter_request=generate_request.prompt_adapter_request)
113
-
114
- async for request_output in results_generator:
115
- await self.socket.send_multipart(
116
- (identity, pickle.dumps(request_output)), copy=False)
117
-
118
- except Exception as e:
119
- await self.socket.send_multipart((identity, pickle.dumps(e)),
120
- copy=False)
121
-
122
- async def check_health(self, identity):
123
- try:
124
- await self.engine.check_health()
125
- await self.socket.send_multipart(
126
- (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
127
-
128
- except Exception as e:
129
- await self.socket.send_multipart((identity, pickle.dumps(e)),
130
- copy=False)
131
-
132
- async def start_profile(self, identity):
133
- logger.info("Starting profiler...")
134
- await self.engine.start_profile()
135
- logger.info("Profiler started.")
136
-
137
- await self.socket.send_multipart((
138
- identity,
139
- pickle.dumps(VLLM_RPC_SUCCESS_STR),
140
- ))
141
-
142
- async def stop_profile(self, identity):
143
- logger.info("Stopping profiler...")
144
- await self.engine.stop_profile()
145
- logger.info("Profiler stopped.")
146
-
147
- await self.socket.send_multipart((
148
- identity,
149
- pickle.dumps(VLLM_RPC_SUCCESS_STR),
150
- ))
151
-
152
- def _make_handler_coro(self, identity,
153
- message: Frame) -> Coroutine[Any, Any, Never]:
154
- """Route the zmq message to the handler coroutine."""
155
-
156
- request = cloudpickle.loads(message.buffer)
157
-
158
- if isinstance(request, RPCGenerateRequest):
159
- return self.generate(identity, request)
160
-
161
- elif isinstance(request, RPCAbortRequest):
162
- return self.abort(identity, request)
163
-
164
- elif isinstance(request, RPCUtilityRequest):
165
- if request in [
166
- RPCUtilityRequest.GET_MODEL_CONFIG,
167
- RPCUtilityRequest.GET_PARALLEL_CONFIG,
168
- RPCUtilityRequest.GET_DECODING_CONFIG,
169
- RPCUtilityRequest.GET_SCHEDULER_CONFIG,
170
- RPCUtilityRequest.GET_LORA_CONFIG
171
- ]:
172
- return self.get_config(identity, request)
173
- elif request == RPCUtilityRequest.DO_LOG_STATS:
174
- return self.do_log_stats(identity)
175
- elif request == RPCUtilityRequest.IS_SERVER_READY:
176
- return self.is_server_ready(identity)
177
- elif request == RPCUtilityRequest.IS_SERVER_HEALTHY:
178
- return self.check_health(identity)
179
- elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
180
- return self.is_tracing_enabled(identity)
181
- elif request == RPCUtilityRequest.START_PROFILE:
182
- return self.start_profile(identity)
183
- elif request == RPCUtilityRequest.STOP_PROFILE:
184
- return self.stop_profile(identity)
185
- else:
186
- raise ValueError(f"Unknown RPCUtilityRequest type: {request}")
187
-
188
- else:
189
- raise ValueError(f"Unknown RPCRequest type: {request}")
190
-
191
- async def run_server_loop(self):
192
- """Inner RPC Server Loop"""
193
-
194
- running_tasks = set()
195
- while True:
196
- # Wait for a request.
197
- identity, message = await self.socket.recv_multipart(copy=False)
198
-
199
- # Process the request async.
200
- task = asyncio.create_task(
201
- self._make_handler_coro(identity, message))
202
-
203
- # We need to keep around a strong reference to the task,
204
- # to avoid the task disappearing mid-execution as running tasks
205
- # can be GC'ed. Below is a common "fire-and-forget" tasks
206
- # https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
207
- running_tasks.add(task)
208
- task.add_done_callback(running_tasks.discard)
209
-
210
-
211
-async def run_server(server: AsyncEngineRPCServer):
212
- # Put the server task into the asyncio loop.
213
- loop = asyncio.get_running_loop()
214
- server_task = loop.create_task(server.run_server_loop())
215
-
216
- # Interruption handling.
217
- def signal_handler() -> None:
218
- # Kill the server on interrupt / terminate
219
- server_task.cancel()
220
-
221
- loop.add_signal_handler(signal.SIGINT, signal_handler)
222
- loop.add_signal_handler(signal.SIGTERM, signal_handler)
223
-
224
- try:
225
- await server_task
226
- except asyncio.CancelledError:
227
- logger.info("vLLM ZMQ RPC Server was interrupted.")
228
- finally:
229
- # Clean up all resources.
230
- server.cleanup()
231
-
232
-
233
-def run_rpc_server(async_engine_args: AsyncEngineArgs,
234
- usage_context: UsageContext, rpc_path: str):
235
-
236
- def signal_handler(*_) -> None:
237
- # Interrupt server on sigterm while initializing
238
- raise KeyboardInterrupt("AsyncEngineRPCServer terminated")
239
-
240
- signal.signal(signal.SIGTERM, signal_handler)
241
-
242
- server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
243
- uvloop.run(run_server(server))
vllm/entrypoints/openai/serving_chat.py
+13
-8
@@ -9,7 +9,7 @@ from typing import Union
9
from fastapi import Request
10
11
from vllm.config import ModelConfig
12
-from vllm.engine.protocol import AsyncEngineClient
12
+from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.chat_utils import (ConversationMessage,
14
apply_hf_chat_template,
15
apply_mistral_chat_template,
@@ -45,7 +45,7 @@ logger = init_logger(__name__)
45
class OpenAIServingChat(OpenAIServing):
46
47
def __init__(self,
48
- async_engine_client: AsyncEngineClient,
48
+ engine_client: EngineClient,
49
model_config: ModelConfig,
50
served_model_names: List[str],
51
response_role: str,
@@ -57,7 +57,7 @@ class OpenAIServingChat(OpenAIServing):
57
return_tokens_as_token_ids: bool = False,
58
enable_auto_tools: bool = False,
59
tool_parser: Optional[str] = None):
60
- super().__init__(async_engine_client=async_engine_client,
60
+ super().__init__(engine_client=engine_client,
61
model_config=model_config,
62
served_model_names=served_model_names,
63
lora_modules=lora_modules,
@@ -105,6 +105,12 @@ class OpenAIServingChat(OpenAIServing):
105
logger.error("Error with model %s", error_check_ret)
106
return error_check_ret
107
108
+ # If the engine is dead, raise the engine's DEAD_ERROR.
109
+ # This is required for the streaming case, where we return a
110
+ # success status before we actually start generating text :).
111
+ if self.engine_client.errored:
112
+ raise self.engine_client.dead_error
113
+
114
try:
115
(
116
lora_request,
@@ -112,8 +118,7 @@ class OpenAIServingChat(OpenAIServing):
118
) = self._maybe_get_adapters(request)
119
120
model_config = self.model_config
115
- tokenizer = await self.async_engine_client.get_tokenizer(
116
- lora_request)
121
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
122
123
conversation, mm_data_future = parse_chat_messages_futures(
124
request.messages, model_config, tokenizer)
@@ -207,8 +212,8 @@ class OpenAIServingChat(OpenAIServing):
212
if mm_data is not None:
213
engine_inputs["multi_modal_data"] = mm_data
214
210
- is_tracing_enabled = (
211
- await self.async_engine_client.is_tracing_enabled())
215
+ is_tracing_enabled = (await
216
+ self.engine_client.is_tracing_enabled())
217
trace_headers = None
218
if is_tracing_enabled and raw_request:
219
trace_headers = extract_trace_headers(raw_request.headers)
@@ -216,7 +221,7 @@ class OpenAIServingChat(OpenAIServing):
221
and contains_trace_headers(raw_request.headers)):
222
log_tracing_disabled_warning()
223
219
- result_generator = self.async_engine_client.generate(
224
+ result_generator = self.engine_client.generate(
225
engine_inputs,
226
sampling_params,
227
request_id,
vllm/entrypoints/openai/serving_completion.py
+13
-8
@@ -8,7 +8,7 @@ from typing import Tuple, Union, cast
8
from fastapi import Request
9
10
from vllm.config import ModelConfig
11
-from vllm.engine.protocol import AsyncEngineClient
11
+from vllm.engine.protocol import EngineClient
12
from vllm.entrypoints.logger import RequestLogger
13
# yapf conflicts with isort for this block
14
# yapf: disable
@@ -43,7 +43,7 @@ class OpenAIServingCompletion(OpenAIServing):
43
44
def __init__(
45
self,
46
- async_engine_client: AsyncEngineClient,
46
+ engine_client: EngineClient,
47
model_config: ModelConfig,
48
served_model_names: List[str],
49
*,
@@ -52,7 +52,7 @@ class OpenAIServingCompletion(OpenAIServing):
52
request_logger: Optional[RequestLogger],
53
return_tokens_as_token_ids: bool = False,
54
):
55
- super().__init__(async_engine_client=async_engine_client,
55
+ super().__init__(engine_client=engine_client,
56
model_config=model_config,
57
served_model_names=served_model_names,
58
lora_modules=lora_modules,
@@ -78,6 +78,12 @@ class OpenAIServingCompletion(OpenAIServing):
78
if error_check_ret is not None:
79
return error_check_ret
80
81
+ # If the engine is dead, raise the engine's DEAD_ERROR.
82
+ # This is required for the streaming case, where we return a
83
+ # success status before we actually start generating text :).
84
+ if self.engine_client.errored:
85
+ raise self.engine_client.dead_error
86
+
87
# Return error for unsupported features.
88
if request.suffix is not None:
89
return self.create_error_response(
@@ -95,8 +101,7 @@ class OpenAIServingCompletion(OpenAIServing):
101
prompt_adapter_request,
102
) = self._maybe_get_adapters(request)
103
98
- tokenizer = await self.async_engine_client.get_tokenizer(
99
- lora_request)
104
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
105
106
guided_decode_logits_processor = (
107
await self._guided_decode_logits_processor(request, tokenizer))
@@ -124,8 +129,8 @@ class OpenAIServingCompletion(OpenAIServing):
129
lora_request=lora_request,
130
prompt_adapter_request=prompt_adapter_request)
131
127
- is_tracing_enabled = (
128
- await self.async_engine_client.is_tracing_enabled())
132
+ is_tracing_enabled = (await
133
+ self.engine_client.is_tracing_enabled())
134
trace_headers = None
135
if is_tracing_enabled:
136
trace_headers = extract_trace_headers(raw_request.headers)
@@ -133,7 +138,7 @@ class OpenAIServingCompletion(OpenAIServing):
138
raw_request.headers):
139
log_tracing_disabled_warning()
140
136
- generator = self.async_engine_client.generate(
141
+ generator = self.engine_client.generate(
142
{"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
143
sampling_params,
144
request_id_item,
vllm/entrypoints/openai/serving_embedding.py
+5
-6
@@ -8,7 +8,7 @@ from fastapi import Request
8
from typing_extensions import assert_never
9
10
from vllm.config import ModelConfig
11
-from vllm.engine.protocol import AsyncEngineClient
11
+from vllm.engine.protocol import EngineClient
12
from vllm.entrypoints.logger import RequestLogger
13
from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
14
EmbeddingResponse,
@@ -71,13 +71,13 @@ class OpenAIServingEmbedding(OpenAIServing):
71
72
def __init__(
73
self,
74
- async_engine_client: AsyncEngineClient,
74
+ engine_client: EngineClient,
75
model_config: ModelConfig,
76
served_model_names: List[str],
77
*,
78
request_logger: Optional[RequestLogger],
79
):
80
- super().__init__(async_engine_client=async_engine_client,
80
+ super().__init__(engine_client=engine_client,
81
model_config=model_config,
82
served_model_names=served_model_names,
83
lora_modules=None,
@@ -118,8 +118,7 @@ class OpenAIServingEmbedding(OpenAIServing):
118
prompt_adapter_request,
119
) = self._maybe_get_adapters(request)
120
121
- tokenizer = await self.async_engine_client.get_tokenizer(
122
- lora_request)
121
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
122
123
pooling_params = request.to_pooling_params()
124
@@ -144,7 +143,7 @@ class OpenAIServingEmbedding(OpenAIServing):
143
"Prompt adapter is not supported "
144
"for embedding models")
145
147
- generator = self.async_engine_client.encode(
146
+ generator = self.engine_client.encode(
147
{"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
148
pooling_params,
149
request_id_item,
vllm/entrypoints/openai/serving_engine.py
+4
-4
@@ -8,7 +8,7 @@ from pydantic import Field
8
from typing_extensions import Annotated
9
10
from vllm.config import ModelConfig
11
-from vllm.engine.protocol import AsyncEngineClient
11
+from vllm.engine.protocol import EngineClient
12
from vllm.entrypoints.logger import RequestLogger
13
# yapf conflicts with isort for this block
14
# yapf: disable
@@ -64,7 +64,7 @@ class OpenAIServing:
64
65
def __init__(
66
self,
67
- async_engine_client: AsyncEngineClient,
67
+ engine_client: EngineClient,
68
model_config: ModelConfig,
69
served_model_names: List[str],
70
*,
@@ -75,7 +75,7 @@ class OpenAIServing:
75
):
76
super().__init__()
77
78
- self.async_engine_client = async_engine_client
78
+ self.engine_client = engine_client
79
self.model_config = model_config
80
self.max_model_len = model_config.max_model_len
81
@@ -159,7 +159,7 @@ class OpenAIServing:
159
async def _guided_decode_logits_processor(
160
self, request: Union[ChatCompletionRequest, CompletionRequest],
161
tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
162
- decoding_config = await self.async_engine_client.get_decoding_config()
162
+ decoding_config = await self.engine_client.get_decoding_config()
163
guided_decoding_backend = request.guided_decoding_backend \
164
or decoding_config.guided_decoding_backend
165
return await get_guided_decoding_logits_processor(
vllm/entrypoints/openai/serving_tokenization.py
+5
-5
@@ -1,7 +1,7 @@
1
from typing import List, Optional, Union
2
3
from vllm.config import ModelConfig
4
-from vllm.engine.protocol import AsyncEngineClient
4
+from vllm.engine.protocol import EngineClient
5
from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
6
apply_mistral_chat_template,
7
load_chat_template,
@@ -29,7 +29,7 @@ class OpenAIServingTokenization(OpenAIServing):
29
30
def __init__(
31
self,
32
- async_engine_client: AsyncEngineClient,
32
+ engine_client: EngineClient,
33
model_config: ModelConfig,
34
served_model_names: List[str],
35
*,
@@ -37,7 +37,7 @@ class OpenAIServingTokenization(OpenAIServing):
37
request_logger: Optional[RequestLogger],
38
chat_template: Optional[str],
39
):
40
- super().__init__(async_engine_client=async_engine_client,
40
+ super().__init__(engine_client=engine_client,
41
model_config=model_config,
42
served_model_names=served_model_names,
43
lora_modules=lora_modules,
@@ -66,7 +66,7 @@ class OpenAIServingTokenization(OpenAIServing):
66
prompt_adapter_request,
67
) = self._maybe_get_adapters(request)
68
69
- tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
69
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
70
71
prompt: Union[str, List[int]]
72
if isinstance(request, TokenizeChatRequest):
@@ -132,7 +132,7 @@ class OpenAIServingTokenization(OpenAIServing):
132
prompt_adapter_request,
133
) = self._maybe_get_adapters(request)
134
135
- tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
135
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
136
137
self._log_inputs(request_id,
138
request.tokens,
vllm/envs.py
+3
-3
@@ -57,7 +57,7 @@ if TYPE_CHECKING:
57
VERBOSE: bool = False
58
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
59
VLLM_TEST_FORCE_FP8_MARLIN: bool = False
60
- VLLM_RPC_GET_DATA_TIMEOUT_MS: int = 5000
60
+ VLLM_RPC_TIMEOUT: int = 10000 # ms
61
VLLM_PLUGINS: Optional[List[str]] = None
62
VLLM_TORCH_PROFILER_DIR: Optional[str] = None
63
VLLM_USE_TRITON_AWQ: bool = False
@@ -393,8 +393,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
393
394
# Time in ms for the zmq client to wait for a response from the backend
395
# server for simple data operations
396
- "VLLM_RPC_GET_DATA_TIMEOUT_MS":
397
- lambda: int(os.getenv("VLLM_RPC_GET_DATA_TIMEOUT_MS", "5000")),
396
+ "VLLM_RPC_TIMEOUT":
397
+ lambda: int(os.getenv("VLLM_RPC_TIMEOUT", "10000")),
398
399
# a list of plugin names to load, separated by commas.
400
# if this is not set, it means all plugins will be loaded
vllm/executor/cpu_executor.py
+1
@@ -106,6 +106,7 @@ class CPUExecutor(ExecutorBase):
106
)) for rank in range(1, world_size)
107
]
108
109
+ self.worker_monitor = None
110
if world_size != 1 or is_async:
111
if is_async:
112
async_worker_list = self.workers + [self.driver_worker]
vllm/executor/multiproc_worker_utils.py
+4
@@ -168,6 +168,8 @@ class ProcessWorkerWrapper:
168
self.tasks[task_id] = future
169
try:
170
self._task_queue.put((task_id, method, args, kwargs))
171
+ except SystemExit:
172
+ raise
173
except BaseException as e:
174
del self.tasks[task_id]
175
raise ChildProcessError("worker died") from e
@@ -222,6 +224,8 @@ def _run_worker_process(
224
try:
225
executor = getattr(worker, method)
226
output = executor(*args, **kwargs)
227
+ except SystemExit:
228
+ raise
229
except KeyboardInterrupt:
230
break
231
except BaseException as e: