@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c7714d856
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core][Bugfix][Perf] Introduce `MQLLMEngine` to avoid `asyncio` OH (#8157)

Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com> Co-authored-by: Simon Mo <simon.mo@hey.com>

Alexander Matveev committed Sep 18, 2024 at 09:56 UTC 7c7714d856eee6fa94aade729b67f00584f72a4c
36 files changed +1467 -1172
.buildkite/test-pipeline.yaml
+3 -1
@@ -43,13 +43,15 @@ steps:
43 fast_check: true
44 source_file_dependencies:
45 - vllm/
46 + - tests/mq_llm_engine
47 - tests/async_engine
48 - tests/test_inputs
49 - tests/multimodal
50 - tests/test_utils
51 - tests/worker
52 commands:
52 - - pytest -v -s async_engine # Async Engine
53 + - pytest -v -s mq_llm_engine # MQLLMEngine
54 + - pytest -v -s async_engine # AsyncLLMEngine
55 - NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
56 - pytest -v -s test_inputs.py
57 - pytest -v -s multimodal
docs/source/dev/profiling/profiling_index.rst
+2 -2
@@ -21,8 +21,8 @@ Traces can be visualized using https://ui.perfetto.dev/.
21 .. tip::
22
23 To stop the profiler - it flushes out all the profile trace files to the directory. This takes time, for example for about 100 requests worth of data for a llama 70b, it takes about 10 minutes to flush out on a H100.
24 - Set the env variable VLLM_RPC_GET_DATA_TIMEOUT_MS to a big number before you start the server. Say something like 30 minutes.
25 - ``export VLLM_RPC_GET_DATA_TIMEOUT_MS=1800000``
24 + Set the env variable VLLM_RPC_TIMEOUT to a big number before you start the server. Say something like 30 minutes.
25 + ``export VLLM_RPC_TIMEOUT=1800000``
26
27 Example commands and usage:
28 ===========================
tests/async_engine/test_openapi_server.py deleted
-106
@@ -1,106 +0,0 @@
1 -import openai # use the official client for correctness check
2 -import pytest
3 -import pytest_asyncio
4 -
5 -from ..utils import VLLM_PATH, RemoteOpenAIServer
6 -
7 -# any model with a chat template should work here
8 -MODEL_NAME = "facebook/opt-125m"
9 -chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
10 -assert chatml_jinja_path.exists()
11 -
12 -
13 -@pytest.fixture(scope="module")
14 -def server():
15 - args = [
16 - # use half precision for speed and memory savings in CI environment
17 - "--dtype",
18 - "float16",
19 - "--max-model-len",
20 - "2048",
21 - "--enforce-eager",
22 - "--chat-template",
23 - str(chatml_jinja_path),
24 - ]
25 -
26 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
27 - yield remote_server
28 -
29 -
30 -@pytest_asyncio.fixture
31 -async def client(server):
32 - async with server.get_async_client() as async_client:
33 - yield async_client
34 -
35 -
36 -@pytest.mark.asyncio
37 -async def test_check_models(client: openai.AsyncOpenAI):
38 - models = await client.models.list()
39 - models = models.data
40 - served_model = models[0]
41 - assert served_model.id == MODEL_NAME
42 - assert all(model.root == MODEL_NAME for model in models)
43 -
44 -
45 -@pytest.mark.asyncio
46 -async def test_single_completion(client: openai.AsyncOpenAI):
47 - completion = await client.completions.create(model=MODEL_NAME,
48 - prompt="Hello, my name is",
49 - max_tokens=5,
50 - temperature=0.0)
51 -
52 - assert completion.id is not None
53 - assert len(completion.choices) == 1
54 - assert len(completion.choices[0].text) >= 5
55 - assert completion.choices[0].finish_reason == "length"
56 - assert completion.usage == openai.types.CompletionUsage(
57 - completion_tokens=5, prompt_tokens=6, total_tokens=11)
58 -
59 - # test using token IDs
60 - completion = await client.completions.create(
61 - model=MODEL_NAME,
62 - prompt=[0, 0, 0, 0, 0],
63 - max_tokens=5,
64 - temperature=0.0,
65 - )
66 - assert len(completion.choices[0].text) >= 5
67 -
68 -
69 -@pytest.mark.asyncio
70 -async def test_single_chat_session(client: openai.AsyncOpenAI):
71 - messages = [{
72 - "role": "system",
73 - "content": "you are a helpful assistant"
74 - }, {
75 - "role": "user",
76 - "content": "what is 1+1?"
77 - }]
78 -
79 - # test single completion
80 - chat_completion = await client.chat.completions.create(model=MODEL_NAME,
81 - messages=messages,
82 - max_tokens=10,
83 - logprobs=True,
84 - top_logprobs=5)
85 - assert chat_completion.id is not None
86 - assert len(chat_completion.choices) == 1
87 -
88 - choice = chat_completion.choices[0]
89 - assert choice.finish_reason == "length"
90 - assert chat_completion.usage == openai.types.CompletionUsage(
91 - completion_tokens=10, prompt_tokens=55, total_tokens=65)
92 -
93 - message = choice.message
94 - assert message.content is not None and len(message.content) >= 10
95 - assert message.role == "assistant"
96 - messages.append({"role": "assistant", "content": message.content})
97 -
98 - # test multi-turn dialogue
99 - messages.append({"role": "user", "content": "express your result in json"})
100 - chat_completion = await client.chat.completions.create(
101 - model=MODEL_NAME,
102 - messages=messages,
103 - max_tokens=10,
104 - )
105 - message = chat_completion.choices[0].message
106 - assert message.content is not None and len(message.content) >= 0
tests/entrypoints/openai/rpc/test_zmq_client.py deleted
-120
@@ -1,120 +0,0 @@
1 -import asyncio
2 -import tempfile
3 -import unittest
4 -import unittest.mock
5 -import uuid
6 -
7 -import pytest
8 -import pytest_asyncio
9 -
10 -from vllm.engine.async_llm_engine import AsyncLLMEngine
11 -from vllm.entrypoints.openai.rpc.client import (AsyncEngineRPCClient,
12 - RPCClientClosedError)
13 -from vllm.entrypoints.openai.rpc.server import AsyncEngineRPCServer
14 -
15 -
16 -@pytest.fixture(scope="function")
17 -def tmp_socket():
18 - with tempfile.TemporaryDirectory() as td:
19 - yield f"ipc://{td}/{uuid.uuid4()}"
20 -
21 -
22 -@pytest_asyncio.fixture(scope="function")
23 -async def dummy_server(tmp_socket, monkeypatch):
24 - dummy_engine = unittest.mock.AsyncMock()
25 -
26 - def dummy_engine_builder(*args, **kwargs):
27 - return dummy_engine
28 -
29 - with monkeypatch.context() as m:
30 - m.setattr(AsyncLLMEngine, "from_engine_args", dummy_engine_builder)
31 - server = AsyncEngineRPCServer(None, None, rpc_path=tmp_socket)
32 -
33 - loop = asyncio.get_running_loop()
34 - server_task = loop.create_task(server.run_server_loop())
35 -
36 - try:
37 - yield server
38 - finally:
39 - server_task.cancel()
40 - server.cleanup()
41 -
42 -
43 -@pytest_asyncio.fixture(scope="function")
44 -async def client(tmp_socket):
45 - client = AsyncEngineRPCClient(rpc_path=tmp_socket)
46 - # Sanity check: the server is connected
47 - await client._wait_for_server_rpc()
48 -
49 - try:
50 - yield client
51 - finally:
52 - client.close()
53 -
54 -
55 -@pytest.mark.asyncio
56 -async def test_client_data_methods_use_timeouts(monkeypatch, dummy_server,
57 - client: AsyncEngineRPCClient):
58 - with monkeypatch.context() as m:
59 - # Make the server _not_ reply with a model config
60 - m.setattr(dummy_server, "get_config", lambda x: None)
61 - m.setattr(client, "_data_timeout", 10)
62 -
63 - # And ensure the task completes anyway
64 - # (client.setup() invokes server.get_config())
65 - client_task = asyncio.get_running_loop().create_task(client.setup())
66 - with pytest.raises(TimeoutError, match="Server didn't reply within"):
67 - await asyncio.wait_for(client_task, timeout=0.05)
68 -
69 -
70 -@pytest.mark.asyncio
71 -async def test_client_aborts_use_timeouts(monkeypatch, dummy_server,
72 - client: AsyncEngineRPCClient):
73 - with monkeypatch.context() as m:
74 - # Hang all abort requests
75 - m.setattr(dummy_server, "abort", lambda x: None)
76 - m.setattr(client, "_data_timeout", 10)
77 -
78 - # The client should suppress timeouts on `abort`s
79 - # and return normally, assuming the server will eventually
80 - # abort the request.
81 - client_task = asyncio.get_running_loop().create_task(
82 - client.abort("test request id"))
83 - await asyncio.wait_for(client_task, timeout=0.05)
84 -
85 -
86 -@pytest.mark.asyncio
87 -async def test_client_data_methods_reraise_exceptions(
88 - monkeypatch, dummy_server, client: AsyncEngineRPCClient):
89 - with monkeypatch.context() as m:
90 - # Make the server raise some random exception
91 - exception = RuntimeError("Client test exception")
92 -
93 - def raiser():
94 - raise exception
95 -
96 - m.setattr(dummy_server.engine, "get_model_config", raiser)
97 - m.setattr(client, "_data_timeout", 10)
98 -
99 - client_task = asyncio.get_running_loop().create_task(client.setup())
100 - # And ensure the task completes, raising the exception
101 - with pytest.raises(RuntimeError, match=str(exception)):
102 - await asyncio.wait_for(client_task, timeout=0.05)
103 -
104 -
105 -@pytest.mark.asyncio
106 -async def test_client_errors_after_closing(monkeypatch, dummy_server,
107 - client: AsyncEngineRPCClient):
108 -
109 - client.close()
110 -
111 - # Healthchecks and generate requests will fail with explicit errors
112 - with pytest.raises(RPCClientClosedError):
113 - await client.check_health()
114 - with pytest.raises(RPCClientClosedError):
115 - async for _ in client.generate(None, None, None):
116 - pass
117 -
118 - # But no-ops like aborting will pass
119 - await client.abort("test-request-id")
120 - await client.do_log_stats()
tests/entrypoints/openai/test_accuracy.py
+25 -31
@@ -18,38 +18,32 @@ TASK = "gsm8k"
18 FILTER = "exact_match,strict-match"
19 RTOL = 0.03
20 EXPECTED_VALUE = 0.58
21 +DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
22 +MORE_ARGS_LIST = [["--enable-chunked-prefill"], ["--num-scheduler-steps", "8"]]
23
24
23 -@pytest.fixture(scope="module")
24 -def server():
25 - args = [
26 - "--max-model-len", "4096", "--enable-chunked-prefill",
27 - "--disable-log-requests", "--enforce-eager"
28 - ]
29 -
30 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31 - yield remote_server
32 -
33 -
34 -@pytest.fixture(scope="module")
35 -def server_data(server):
36 - return {
37 - "url": f"{server.url_for('v1')}/completions",
38 - }
25 +@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
26 +def test_lm_eval_accuracy(more_args):
27 + args = list(DEFAULT_ARGS)
28 + args.extend(more_args)
29
30 + print(f"Running with: {args}")
31
41 -def test_lm_eval_accuracy(server_data):
42 - model_args = (f"model={MODEL_NAME},"
43 - f"base_url={server_data['url']},"
44 - f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False")
45 -
46 - results = lm_eval.simple_evaluate(
47 - model="local-completions",
48 - model_args=model_args,
49 - tasks=TASK,
50 - )
51 -
52 - measured_value = results["results"][TASK][FILTER]
53 - assert (measured_value - RTOL < EXPECTED_VALUE
54 - and measured_value + RTOL > EXPECTED_VALUE
55 - ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
32 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
33 + url = f"{remote_server.url_for('v1')}/completions"
34 +
35 + model_args = (
36 + f"model={MODEL_NAME},"
37 + f"base_url={url},"
38 + f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False")
39 +
40 + results = lm_eval.simple_evaluate(
41 + model="local-completions",
42 + model_args=model_args,
43 + tasks=TASK,
44 + )
45 +
46 + measured_value = results["results"][TASK][FILTER]
47 + assert (measured_value - RTOL < EXPECTED_VALUE
48 + and measured_value + RTOL > EXPECTED_VALUE
49 + ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
tests/entrypoints/openai/test_chat_template.py renamed
+1 -1
@@ -5,7 +5,7 @@ from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
5 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
6 from vllm.transformers_utils.tokenizer import get_tokenizer
7
8 -from ..utils import VLLM_PATH
8 +from ...utils import VLLM_PATH
9
10 chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
11 assert chatml_jinja_path.exists()
tests/entrypoints/openai/test_mp_api_server.py deleted
-40
@@ -1,40 +0,0 @@
1 -import time
2 -
3 -import pytest
4 -
5 -from vllm.entrypoints.openai.api_server import build_async_engine_client
6 -from vllm.entrypoints.openai.cli_args import make_arg_parser
7 -from vllm.utils import FlexibleArgumentParser
8 -
9 -
10 -@pytest.mark.asyncio
11 -async def test_mp_crash_detection():
12 -
13 - parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
14 - parser = make_arg_parser(parser)
15 - args = parser.parse_args([])
16 - # use an invalid tensor_parallel_size to trigger the
17 - # error in the server
18 - args.tensor_parallel_size = 65536
19 -
20 - start = time.perf_counter()
21 - async with build_async_engine_client(args):
22 - pass
23 - end = time.perf_counter()
24 -
25 - assert end - start < 60, ("Expected vLLM to gracefully shutdown in <60s "
26 - "if there is an error in the startup.")
27 -
28 -
29 -@pytest.mark.asyncio
30 -async def test_mp_cuda_init():
31 - # it should not crash, when cuda is initialized
32 - # in the API server process
33 - import torch
34 - torch.cuda.init()
35 - parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
36 - parser = make_arg_parser(parser)
37 - args = parser.parse_args([])
38 -
39 - async with build_async_engine_client(args):
40 - pass
tests/entrypoints/openai/test_serving_chat.py
+3 -2
@@ -4,7 +4,7 @@ from dataclasses import dataclass
4 from unittest.mock import MagicMock
5
6 from vllm.config import MultiModalConfig
7 -from vllm.engine.async_llm_engine import AsyncLLMEngine
7 +from vllm.engine.multiprocessing.client import MQLLMEngineClient
8 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
10 from vllm.transformers_utils.tokenizer import get_tokenizer
@@ -52,8 +52,9 @@ def test_async_serving_chat_init():
52
53
54 def test_serving_chat_should_set_correct_max_tokens():
55 - mock_engine = MagicMock(spec=AsyncLLMEngine)
55 + mock_engine = MagicMock(spec=MQLLMEngineClient)
56 mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
57 + mock_engine.errored = False
58
59 serving_chat = OpenAIServingChat(mock_engine,
60 MockModelConfig(),
tests/entrypoints/openai/test_serving_engine.py
+2 -2
@@ -4,7 +4,7 @@ from unittest.mock import MagicMock
4 import pytest
5
6 from vllm.config import ModelConfig
7 -from vllm.engine.protocol import AsyncEngineClient
7 +from vllm.engine.protocol import EngineClient
8 from vllm.entrypoints.openai.protocol import (ErrorResponse,
9 LoadLoraAdapterRequest,
10 UnloadLoraAdapterRequest)
@@ -18,7 +18,7 @@ LORA_UNLOADING_SUCCESS_MESSAGE = (
18
19
20 async def _async_serving_engine_init():
21 - mock_engine_client = MagicMock(spec=AsyncEngineClient)
21 + mock_engine_client = MagicMock(spec=EngineClient)
22 mock_model_config = MagicMock(spec=ModelConfig)
23 # Set the max_model_len attribute to avoid missing attribute
24 mock_model_config.max_model_len = 2048
tests/entrypoints/openai/test_shutdown.py
+1 -1
@@ -44,5 +44,5 @@ async def test_shutdown_on_engine_failure(tmp_path):
44 prompt="Hello, my name is")
45
46 # Now the server should shut down
47 - return_code = remote_server.proc.wait(timeout=3)
47 + return_code = remote_server.proc.wait(timeout=8)
48 assert return_code is not None
tests/mq_llm_engine/__init__.py renamed
tests/mq_llm_engine/test_abort.py new
+67
@@ -0,0 +1,67 @@
1 +"""Test that aborting is handled properly."""
2 +
3 +import asyncio
4 +import tempfile
5 +import uuid
6 +
7 +import pytest
8 +
9 +from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
10 +from vllm.engine.arg_utils import AsyncEngineArgs
11 +
12 +MODEL = "google/gemma-1.1-2b-it"
13 +ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
14 +RAISED_ERROR = KeyError
15 +RAISED_VALUE = "foo"
16 +EXPECTED_TOKENS = 250
17 +
18 +
19 +@pytest.fixture(scope="function")
20 +def tmp_socket():
21 + with tempfile.TemporaryDirectory() as td:
22 + yield f"ipc://{td}/{uuid.uuid4()}"
23 +
24 +
25 +@pytest.mark.asyncio
26 +async def test_abort(tmp_socket):
27 + with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
28 + ipc_path=tmp_socket) as engine:
29 +
30 + client = await engine.make_client()
31 +
32 + request_id_to_be_aborted = "request-aborted"
33 + request_ids_a = [f"request-a-{idx}" for idx in range(10)]
34 + request_ids_b = [f"request-b-{idx}" for idx in range(10)]
35 +
36 + # Requests started before one to be aborted.
37 + tasks = []
38 + for request_id in request_ids_a:
39 + tasks.append(
40 + asyncio.create_task(
41 + generate(client, request_id, EXPECTED_TOKENS)))
42 +
43 + # Aborted.
44 + task_aborted = asyncio.create_task(
45 + generate(client, request_id_to_be_aborted, EXPECTED_TOKENS))
46 +
47 + # Requests started after one to be aborted.
48 + for request_id in request_ids_b:
49 + tasks.append(
50 + asyncio.create_task(
51 + generate(client, request_id, EXPECTED_TOKENS)))
52 +
53 + # Actually abort.
54 + await asyncio.sleep(0.5)
55 + await client.abort(request_id_to_be_aborted)
56 +
57 + # Confirm that we got all the EXPECTED tokens from the requests.
58 + for task in tasks:
59 + count, request_id = await task
60 + assert count == EXPECTED_TOKENS, (
61 + f"{request_id} generated only {count} tokens")
62 +
63 + # Cancel task (this will hang indefinitely if not).
64 + task_aborted.cancel()
65 +
66 + # Shutdown.
67 + client.close()
tests/mq_llm_engine/test_error_handling.py new
+244
@@ -0,0 +1,244 @@
1 +"""Test that various errors are handled properly."""
2 +
3 +import asyncio
4 +import tempfile
5 +import time
6 +import uuid
7 +from unittest.mock import Mock
8 +
9 +import pytest
10 +
11 +from tests.mq_llm_engine.utils import RemoteMQLLMEngine
12 +from vllm import SamplingParams
13 +from vllm.engine.arg_utils import AsyncEngineArgs
14 +from vllm.engine.llm_engine import LLMEngine
15 +from vllm.engine.multiprocessing import MQEngineDeadError
16 +from vllm.engine.multiprocessing.engine import MQLLMEngine
17 +from vllm.entrypoints.openai.api_server import build_async_engine_client
18 +from vllm.entrypoints.openai.cli_args import make_arg_parser
19 +from vllm.lora.request import LoRARequest
20 +from vllm.usage.usage_lib import UsageContext
21 +from vllm.utils import FlexibleArgumentParser
22 +
23 +MODEL = "google/gemma-1.1-2b-it"
24 +ENGINE_ARGS = AsyncEngineArgs(model=MODEL)
25 +RAISED_ERROR = KeyError
26 +RAISED_VALUE = "foo"
27 +
28 +
29 +@pytest.fixture(scope="function")
30 +def tmp_socket():
31 + with tempfile.TemporaryDirectory() as td:
32 + yield f"ipc://{td}/{uuid.uuid4()}"
33 +
34 +
35 +def run_with_evil_forward(engine_args: AsyncEngineArgs, ipc_path: str):
36 + # Make engine.
37 + engine = MQLLMEngine.from_engine_args(
38 + engine_args=engine_args,
39 + usage_context=UsageContext.UNKNOWN_CONTEXT,
40 + ipc_path=ipc_path)
41 +
42 + # Raise error during first forward pass.
43 + engine.engine.model_executor.execute_model = Mock(
44 + side_effect=RAISED_ERROR(RAISED_VALUE))
45 +
46 + # Run engine.
47 + engine.start()
48 +
49 +
50 +@pytest.mark.asyncio
51 +async def test_evil_forward(tmp_socket):
52 + with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
53 + ipc_path=tmp_socket,
54 + run_fn=run_with_evil_forward) as engine:
55 +
56 + client = await engine.make_client()
57 +
58 + # Server should be healthy after initial probe.
59 + await asyncio.sleep(2.0)
60 + await client.check_health()
61 +
62 + # Throws an error in first forward pass.
63 + with pytest.raises(RAISED_ERROR):
64 + async for _ in client.generate(inputs="Hello my name is",
65 + sampling_params=SamplingParams(),
66 + request_id=uuid.uuid4()):
67 + pass
68 + assert client.errored
69 +
70 + # Engine is errored, should get ENGINE_DEAD_ERROR.
71 + with pytest.raises(MQEngineDeadError):
72 + async for _ in client.generate(inputs="Hello my name is",
73 + sampling_params=SamplingParams(),
74 + request_id=uuid.uuid4()):
75 + pass
76 + assert client.errored
77 +
78 + await asyncio.sleep(1.0)
79 + with pytest.raises(RAISED_ERROR):
80 + await client.check_health()
81 + assert client.errored
82 +
83 + # Shutdown.
84 + client.close()
85 +
86 +
87 +def run_with_evil_model_executor_health(engine_args: AsyncEngineArgs,
88 + ipc_path: str):
89 + # Make engine.
90 + engine = MQLLMEngine.from_engine_args(
91 + engine_args=engine_args,
92 + usage_context=UsageContext.UNKNOWN_CONTEXT,
93 + ipc_path=ipc_path)
94 +
95 + # Raise error during first forward pass.
96 + engine.engine.model_executor.check_health = Mock(side_effect=RAISED_ERROR)
97 +
98 + # Run engine.
99 + engine.start()
100 +
101 +
102 +@pytest.mark.asyncio
103 +async def test_failed_health_check(tmp_socket):
104 + with RemoteMQLLMEngine(
105 + engine_args=ENGINE_ARGS,
106 + ipc_path=tmp_socket,
107 + run_fn=run_with_evil_model_executor_health) as engine:
108 +
109 + client = await engine.make_client()
110 + assert client.is_running
111 +
112 + # Health probe should throw RAISED_ERROR.
113 + await asyncio.sleep(15.)
114 +
115 + with pytest.raises(RAISED_ERROR):
116 + await client.check_health()
117 + assert client.errored
118 +
119 + # Generate call should throw ENGINE_DEAD_ERROR
120 + with pytest.raises(MQEngineDeadError):
121 + async for _ in client.generate(inputs="Hello my name is",
122 + sampling_params=SamplingParams(),
123 + request_id=uuid.uuid4()):
124 + pass
125 +
126 + client.close()
127 +
128 +
129 +def run_with_evil_abort(engine_args: AsyncEngineArgs, ipc_path: str):
130 + # Make engine.
131 + engine = MQLLMEngine.from_engine_args(
132 + engine_args=engine_args,
133 + usage_context=UsageContext.UNKNOWN_CONTEXT,
134 + ipc_path=ipc_path)
135 +
136 + # Raise error during abort call.
137 + engine.engine.abort_request = Mock(side_effect=RAISED_ERROR)
138 +
139 + # Run engine.
140 + engine.start()
141 +
142 +
143 +@pytest.mark.asyncio
144 +async def test_failed_abort(tmp_socket):
145 + with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
146 + ipc_path=tmp_socket,
147 + run_fn=run_with_evil_abort) as engine:
148 +
149 + client = await engine.make_client()
150 + assert client.is_running
151 +
152 + # Firsh check health should work.
153 + await client.check_health()
154 +
155 + # Trigger an abort on the client side.
156 + async def bad_abort_after_2s():
157 + await asyncio.sleep(2.0)
158 + await client.abort(request_id="foo")
159 +
160 + # Trigger an abort in 2s from now.
161 + abort_task = asyncio.create_task(bad_abort_after_2s())
162 +
163 + # Exception in abort() will happen during this generation.
164 + # This will kill the engine and should return ENGINE_DEAD_ERROR
165 + # with reference to the original KeyError("foo")
166 + with pytest.raises(MQEngineDeadError) as execinfo:
167 + async for _ in client.generate(
168 + inputs="Hello my name is",
169 + sampling_params=SamplingParams(max_tokens=2000),
170 + request_id=uuid.uuid4()):
171 + pass
172 + assert "KeyError" in repr(execinfo.value)
173 + assert client.errored
174 +
175 + await abort_task
176 +
177 + # This should raise the original error.
178 + with pytest.raises(RAISED_ERROR):
179 + await client.check_health()
180 +
181 + client.close()
182 +
183 +
184 +@pytest.mark.asyncio
185 +async def test_bad_request(tmp_socket):
186 + with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
187 + ipc_path=tmp_socket) as engine:
188 +
189 + client = await engine.make_client()
190 +
191 + # Invalid request should fail, but not crash the server.
192 + with pytest.raises(ValueError):
193 + async for _ in client.generate(inputs="Hello my name is",
194 + sampling_params=SamplingParams(),
195 + request_id="abcd-1",
196 + lora_request=LoRARequest(
197 + "invalid-lora", 1,
198 + "invalid-path")):
199 + pass
200 +
201 + # This request should be okay.
202 + async for _ in client.generate(inputs="Hello my name is",
203 + sampling_params=SamplingParams(),
204 + request_id="abcd-2"):
205 + pass
206 +
207 + # Shutdown.
208 + client.close()
209 +
210 +
211 +@pytest.mark.asyncio
212 +async def test_mp_crash_detection(monkeypatch):
213 +
214 + parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
215 + parser = make_arg_parser(parser)
216 + args = parser.parse_args([])
217 +
218 + # When LLMEngine is loaded, it will crash.
219 + def mock_init():
220 + raise ValueError
221 +
222 + monkeypatch.setattr(LLMEngine, "__init__", mock_init)
223 +
224 + start = time.perf_counter()
225 + async with build_async_engine_client(args):
226 + pass
227 + end = time.perf_counter()
228 +
229 + assert end - start < 60, ("Expected vLLM to gracefully shutdown in <60s "
230 + "if there is an error in the startup.")
231 +
232 +
233 +@pytest.mark.asyncio
234 +async def test_mp_cuda_init():
235 + # it should not crash, when cuda is initialized
236 + # in the API server process
237 + import torch
238 + torch.cuda.init()
239 + parser = FlexibleArgumentParser(description="vLLM's remote OpenAI server.")
240 + parser = make_arg_parser(parser)
241 + args = parser.parse_args([])
242 +
243 + async with build_async_engine_client(args):
244 + pass
tests/mq_llm_engine/test_load.py new
+57
@@ -0,0 +1,57 @@
1 +"""Test that the MQLLMEngine is able to handle 10k concurrent requests."""
2 +
3 +import asyncio
4 +import tempfile
5 +import uuid
6 +
7 +import pytest
8 +
9 +from tests.mq_llm_engine.utils import RemoteMQLLMEngine, generate
10 +from vllm.engine.arg_utils import AsyncEngineArgs
11 +
12 +MODEL = "google/gemma-1.1-2b-it"
13 +NUM_EXPECTED_TOKENS = 10
14 +NUM_REQUESTS = 10000
15 +
16 +# Scenarios to test for num generated token.
17 +ENGINE_ARGS = AsyncEngineArgs(model=MODEL, disable_log_requests=True)
18 +
19 +
20 +@pytest.fixture(scope="function")
21 +def tmp_socket():
22 + with tempfile.TemporaryDirectory() as td:
23 + yield f"ipc://{td}/{uuid.uuid4()}"
24 +
25 +
26 +@pytest.mark.asyncio
27 +async def test_load(tmp_socket):
28 + with RemoteMQLLMEngine(engine_args=ENGINE_ARGS,
29 + ipc_path=tmp_socket) as engine:
30 +
31 + client = await engine.make_client()
32 +
33 + request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
34 +
35 + # Create concurrent requests.
36 + tasks = []
37 + for request_id in request_ids:
38 + tasks.append(
39 + asyncio.create_task(
40 + generate(client, request_id, NUM_EXPECTED_TOKENS)))
41 +
42 + # Confirm that we got all the EXPECTED tokens from the requests.
43 + failed_request_id = None
44 + tokens = None
45 + for task in tasks:
46 + num_generated_tokens, request_id = await task
47 + if (num_generated_tokens != NUM_EXPECTED_TOKENS
48 + and failed_request_id is None):
49 + failed_request_id = request_id
50 + tokens = num_generated_tokens
51 +
52 + assert failed_request_id is None, (
53 + f"{failed_request_id} generated {tokens} but "
54 + f"expected {NUM_EXPECTED_TOKENS}")
55 +
56 + # Shutdown.
57 + client.close()
tests/mq_llm_engine/utils.py new
+78
@@ -0,0 +1,78 @@
1 +import asyncio
2 +import multiprocessing
3 +from typing import Callable, Tuple, Union
4 +
5 +from vllm import SamplingParams
6 +from vllm.engine.arg_utils import AsyncEngineArgs
7 +from vllm.engine.multiprocessing.client import MQLLMEngineClient
8 +from vllm.engine.multiprocessing.engine import MQLLMEngine
9 +from vllm.outputs import RequestOutput
10 +from vllm.usage.usage_lib import UsageContext
11 +
12 +
13 +async def generate(
14 + client: MQLLMEngineClient,
15 + request_id: str,
16 + num_tokens: int,
17 + return_output: bool = False) -> Union[RequestOutput, Tuple[int, str]]:
18 +
19 + final_output = None
20 + count = 0
21 + async for out in client.generate(
22 + request_id=request_id,
23 + inputs="Hello my name is Robert and",
24 + sampling_params=SamplingParams(max_tokens=num_tokens,
25 + temperature=0)):
26 +
27 + count += 1
28 + final_output = out
29 + await asyncio.sleep(0.)
30 +
31 + if return_output:
32 + return final_output
33 +
34 + # Confirm we generated all the tokens we expected.
35 + return count, request_id
36 +
37 +
38 +def run_normal(engine_args: AsyncEngineArgs, ipc_path: str):
39 + # Make engine.
40 + engine = MQLLMEngine.from_engine_args(
41 + engine_args=engine_args,
42 + usage_context=UsageContext.UNKNOWN_CONTEXT,
43 + ipc_path=ipc_path)
44 +
45 + # Run engine.
46 + engine.start()
47 +
48 +
49 +class RemoteMQLLMEngine:
50 +
51 + def __init__(self,
52 + engine_args: AsyncEngineArgs,
53 + ipc_path: str,
54 + run_fn: Callable = run_normal) -> None:
55 +
56 + self.engine_args = engine_args
57 + self.ipc_path = ipc_path
58 + context = multiprocessing.get_context("spawn")
59 + self.proc = context.Process(target=run_fn,
60 + args=(engine_args, ipc_path))
61 + self.proc.start()
62 +
63 + def __enter__(self):
64 + return self
65 +
66 + def __exit__(self, exc_type, exc_value, traceback):
67 + self.proc.kill()
68 +
69 + async def make_client(self) -> MQLLMEngineClient:
70 + engine_config = self.engine_args.create_engine_config()
71 + client = MQLLMEngineClient(self.ipc_path, engine_config)
72 + while True:
73 + try:
74 + await client.setup()
75 + break
76 + except TimeoutError:
77 + assert self.proc.is_alive()
78 + return client
tests/tpu/test_custom_dispatcher.py
+7
@@ -1,5 +1,12 @@
1 +import os
2 +
3 from ..utils import compare_two_settings
4
5 +# --enforce-eager on TPU causes graph compilation
6 +# this times out default Health Check in the MQLLMEngine,
7 +# so we set the timeout here to 30s
8 +os.environ["VLLM_RPC_TIMEOUT"] = "30000"
9 +
10
11 def test_custom_dispatcher():
12 compare_two_settings("google/gemma-2b",
tests/utils.py
+1 -1
@@ -119,7 +119,7 @@ class RemoteOpenAIServer:
119 def __exit__(self, exc_type, exc_value, traceback):
120 self.proc.terminate()
121 try:
122 - self.proc.wait(3)
122 + self.proc.wait(8)
123 except subprocess.TimeoutExpired:
124 # force kill if needed
125 self.proc.kill()
vllm/engine/async_llm_engine.py
+6 -3
@@ -601,9 +601,12 @@ class AsyncLLMEngine:
601 return self._errored_with is not None
602
603 @property
604 - def limit_concurrency(self) -> Optional[int]:
605 - """Maximum number of concurrently running requests."""
606 - return None
604 + def dead_error(self) -> BaseException:
605 + return AsyncEngineDeadError(
606 + "Background loop is not running. If it was running, "
607 + "inspect the output to find the stacktrace of the "
608 + "error that caused the background loop to stop "
609 + "(AsyncEngineDeadError).")
610
611 def set_errored(self, exc: Exception) -> None:
612 self._errored_with = exc
vllm/engine/llm_engine.py
+1
@@ -1289,6 +1289,7 @@ class LLMEngine:
1289 # torch.distributed ops which may otherwise timeout, and unblocks
1290 # the RPC thread in the workers so that they can process any other
1291 # queued control plane messages, such as add/remove lora adapters.
1292 + logger.debug("Stopping remote worker execution loop.")
1293 self.model_executor.stop_remote_worker_execution_loop()
1294
1295 return ctx.request_outputs
vllm/engine/multiprocessing/__init__.py new
+73
@@ -0,0 +1,73 @@
1 +from dataclasses import dataclass
2 +from enum import Enum
3 +from typing import List, Mapping, Optional, Union
4 +
5 +from vllm.inputs import PromptInputs
6 +from vllm.lora.request import LoRARequest
7 +from vllm.outputs import RequestOutput
8 +from vllm.prompt_adapter.request import PromptAdapterRequest
9 +from vllm.sampling_params import SamplingParams
10 +
11 +VLLM_RPC_SUCCESS_STR = "SUCCESS"
12 +
13 +IPC_INPUT_EXT = "_input_socket"
14 +IPC_OUTPUT_EXT = "_output_socket"
15 +IPC_HEALTH_EXT = "_health_socket"
16 +IPC_DATA_EXT = "_data_socket"
17 +
18 +
19 +class MQEngineDeadError(RuntimeError):
20 + pass
21 +
22 +
23 +@dataclass
24 +class RPCGenerateRequest:
25 + inputs: PromptInputs
26 + sampling_params: SamplingParams
27 + request_id: str
28 + lora_request: Optional[LoRARequest] = None
29 + trace_headers: Optional[Mapping[str, str]] = None
30 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
31 +
32 +
33 +@dataclass
34 +class RPCError:
35 + request_id: Optional[str]
36 + is_engine_errored: bool
37 + exception: BaseException
38 +
39 +
40 +@dataclass
41 +class RPCAbortRequest:
42 + request_id: str
43 +
44 +
45 +class RPCHealthRequest:
46 + pass
47 +
48 +
49 +class RPCStartupRequest(Enum):
50 + IS_SERVER_READY = 1
51 +
52 +
53 +@dataclass
54 +class RPCStartupResponse:
55 + tracing_enabled: bool
56 +
57 +
58 +RPC_REQUEST_T = Union[RPCGenerateRequest, RPCAbortRequest, RPCHealthRequest,
59 + RPCStartupRequest]
60 +
61 +REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
62 +
63 +
64 +def ENGINE_DEAD_ERROR(
65 + error: Optional[BaseException] = None) -> MQEngineDeadError:
66 + if error is None:
67 + return MQEngineDeadError(
68 + "Engine loop is not running. Inspect the stacktrace to "
69 + "find the original error")
70 +
71 + return MQEngineDeadError(
72 + "Engine loop is not running. Inspect the stacktrace to "
73 + f"find the original error: {repr(error)}.")
vllm/engine/multiprocessing/client.py new
+452
@@ -0,0 +1,452 @@
1 +import asyncio
2 +import copy
3 +import pickle
4 +from contextlib import contextmanager, suppress
5 +from typing import (Any, AsyncGenerator, Dict, Iterator, Mapping, Optional,
6 + Union)
7 +
8 +import cloudpickle
9 +import zmq
10 +import zmq.asyncio
11 +from zmq import Frame # type: ignore[attr-defined]
12 +from zmq.asyncio import Socket
13 +
14 +from vllm.config import DecodingConfig, EngineConfig, ModelConfig
15 +from vllm.engine.arg_utils import AsyncEngineArgs
16 +# yapf conflicts with isort for this block
17 +# yapf: disable
18 +from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
19 + IPC_HEALTH_EXT, IPC_INPUT_EXT,
20 + IPC_OUTPUT_EXT, RPC_REQUEST_T,
21 + VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
22 + RPCError, RPCGenerateRequest,
23 + RPCHealthRequest, RPCStartupRequest,
24 + RPCStartupResponse)
25 +# yapf: enable
26 +from vllm.envs import VLLM_RPC_TIMEOUT
27 +from vllm.inputs import PromptInputs
28 +from vllm.logger import init_logger
29 +from vllm.lora.request import LoRARequest
30 +from vllm.outputs import EmbeddingRequestOutput, RequestOutput
31 +from vllm.prompt_adapter.request import PromptAdapterRequest
32 +from vllm.sampling_params import SamplingParams
33 +from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
34 +
35 +logger = init_logger(__name__)
36 +
37 +
38 +class MQClientClosedError(Exception):
39 + """Exception class raised when the client is used post-close.
40 +
41 + The client can be closed, which closes the ZMQ context. This normally
42 + happens on server shutdown. In some cases, methods like abort and
43 + do_log_stats will still be called and then try to open a socket, which
44 + causes a ZMQError and creates a huge stack trace.
45 + So, we throw this error such that we can suppress it.
46 + """
47 +
48 +
49 +class MQLLMEngineClient:
50 + """A client wrapper for MQLLMEngine that conforms to the
51 + EngineClient protocol.
52 +
53 + MQLLMEngine and MQLLMEngineClient are intended to run in separate
54 + processes communicating via zeromq ipc sockets.
55 +
56 + The entrypoint to MQLLMEngineClient is through the generate()
57 + method. On generate() MQLLMEngine does three things:
58 + - Creates an asyncio output queue
59 + - Sends a RPCGenerateRequest to the MQLLMEngine via zmq
60 + - Pulls RequestOutputs from its queue and yields them
61 +
62 + MQLLMEngine runs two background loops:
63 + - output_loop: the output loop pulls List[RequestOutput]
64 + from the MQLLMEngine via zmq (each list is the output
65 + of one engine_step in the LLMEngine). It then parses
66 + the list and pushes individual request_outputs into
67 + the corresponding output_queue such that they can be
68 + consumed by the .generate() method.
69 + - health_loop: the health loop queries the health socket
70 + every N seconds, confirming the engine is healthy
71 + """
72 +
73 + def __init__(self, ipc_path: str, engine_config: EngineConfig):
74 + self.context = zmq.asyncio.Context()
75 + self._errored_with: Optional[BaseException] = None
76 +
77 + # Get the configs.
78 + self.model_config = engine_config.model_config
79 + self.decoding_config = engine_config.decoding_config
80 +
81 + # Create the tokenizer group.
82 + self.tokenizer = init_tokenizer_from_configs(
83 + model_config=self.model_config,
84 + scheduler_config=engine_config.scheduler_config,
85 + parallel_config=engine_config.parallel_config,
86 + enable_lora=bool(engine_config.lora_config),
87 + )
88 +
89 + # Send RPCGenerateRequest to the MQLLMEngine.
90 + self.input_socket: Socket = self.context.socket(zmq.constants.PUSH)
91 + self.input_socket.connect(f"{ipc_path}{IPC_INPUT_EXT}")
92 +
93 + # Receive streams of RequestOutput from the MQLLMEngine.
94 + self.output_socket: Socket = self.context.socket(zmq.constants.PULL)
95 + self.output_socket.connect(f"{ipc_path}{IPC_OUTPUT_EXT}")
96 +
97 + # IPC path for ack of check_health requests.
98 + self.health_socket: Socket = self.context.socket(zmq.constants.PULL)
99 + self.health_socket.connect(f"{ipc_path}{IPC_HEALTH_EXT}")
100 +
101 + # IPC path for the data socket.
102 + self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
103 +
104 + # Stream for each individual request.
105 + self.output_queues: Dict[str, asyncio.Queue] = {}
106 + self.output_loop = asyncio.create_task(self.run_output_handler_loop())
107 +
108 + # Loop to check health of the LLMEngine periodically.
109 + # Started after the MQLLMEngine is ready.
110 + self.health_loop: Optional[asyncio.Task] = None
111 +
112 + @staticmethod
113 + def is_unsupported_config(engine_args: AsyncEngineArgs):
114 + if engine_args.pipeline_parallel_size > 1:
115 + return True
116 +
117 + is_embedding = ModelConfig(
118 + model=engine_args.model,
119 + revision=engine_args.revision,
120 + tokenizer=engine_args.model,
121 + tokenizer_mode="auto",
122 + trust_remote_code=engine_args.trust_remote_code,
123 + quantization=engine_args.quantization,
124 + seed=0,
125 + dtype="auto").embedding_mode
126 +
127 + return is_embedding
128 +
129 + @contextmanager
130 + def get_data_socket(self) -> Iterator[Socket]:
131 + socket = self.context.socket(zmq.constants.DEALER)
132 + try:
133 + socket.connect(self.data_ipc_path)
134 + yield socket
135 + finally:
136 + socket.close(linger=0)
137 +
138 + async def run_check_health_loop(self, timeout: int):
139 + """Background loop that continually probes the RPCServer for health.
140 +
141 + The loop sends CHECK_HEALTH requests to the INPUT_SOCKET, which
142 + the MQLLMEngine server is blocking on.
143 +
144 + The Server replies on the HEALTH_SOCKET (rather than on the
145 + OUTPUT_SOCKET such that the messages are not intermingled with
146 + output streaming).
147 + """
148 +
149 + try:
150 + while True:
151 + if await self.health_socket.poll(timeout=timeout) == 0:
152 + # Wakeup every N seconds and do a health probe.
153 + await self._send_one_way_rpc_request(
154 + RPCHealthRequest(), self.input_socket)
155 +
156 + # Wait for ack from the health socket.
157 + await self._await_ack(error_message="Health check failed.",
158 + socket=self.health_socket)
159 + else:
160 + # Server sent a health status message unprompted.
161 + await self._check_success(
162 + error_message="Health check failed.",
163 + socket=self.health_socket)
164 +
165 + logger.debug("Health probe successful.")
166 +
167 + except asyncio.CancelledError:
168 + logger.debug("Shutting down MQLLMEngineClient check health loop.")
169 +
170 + except Exception as e:
171 + self._set_errored(e)
172 +
173 + async def run_output_handler_loop(self):
174 + """Get RequestOutputs from Engine and stream to Request Queues"""
175 +
176 + try:
177 + while True:
178 + # Poll, checking for ENGINE_DEAD
179 + while await self.output_socket.poll(timeout=VLLM_RPC_TIMEOUT
180 + ) == 0:
181 + logger.debug("Waiting for output from MQLLMEngine.")
182 +
183 + # If errored, alert all running requests.
184 + if self.errored:
185 + for queue_j in tuple(self.output_queues.values()):
186 + queue_j.put_nowait(
187 + ENGINE_DEAD_ERROR(self._errored_with))
188 + return
189 +
190 + message: Frame = await self.output_socket.recv(copy=False)
191 + request_outputs = pickle.loads(message.buffer)
192 +
193 + is_error = isinstance(request_outputs,
194 + (BaseException, RPCError))
195 + if is_error:
196 + if isinstance(request_outputs, RPCError):
197 + rpc_error: RPCError = request_outputs
198 + request_id = rpc_error.request_id
199 + exception = rpc_error.exception
200 + is_engine_errored = rpc_error.is_engine_errored
201 + else:
202 + # MPLLMEngine should always return an RPCError to
203 + # the output_socket when an issue arises.
204 + # If we are here, we are in a bad state and
205 + # should shut down the server.
206 + error: BaseException = request_outputs
207 + logger.error(
208 + "Received Exception %s rather than RPCError from "
209 + "MPLLMEngine. This should never happen.", error)
210 + request_id = None
211 + exception = error
212 + is_engine_errored = True
213 +
214 + # Set to error state only on engine critical error
215 + # (and record only the first one)
216 + if is_engine_errored and not self._errored_with:
217 + self._errored_with = exception
218 +
219 + if request_id is None:
220 + for queue_i in tuple(self.output_queues.values()):
221 + queue_i.put_nowait(exception)
222 + else:
223 + queue = self.output_queues.get(request_id)
224 + if queue is not None:
225 + queue.put_nowait(exception)
226 + else:
227 + # Put each output into the appropriate steam.
228 + for request_output in request_outputs:
229 + queue = self.output_queues.get(
230 + request_output.request_id)
231 + if queue is not None:
232 + queue.put_nowait(request_output)
233 +
234 + except asyncio.CancelledError:
235 + logger.debug("Shutting down MQLLMEngineClient output handler.")
236 +
237 + async def setup(self):
238 + """Setup the client before it starts sending server requests."""
239 +
240 + with self.get_data_socket() as socket:
241 + # Wait until server is ready.
242 + response = await self._wait_for_server_rpc(socket)
243 +
244 + self.tracing_flag = response.tracing_enabled
245 +
246 + # Start health_loop.
247 + self.health_loop = asyncio.create_task(
248 + self.run_check_health_loop(timeout=VLLM_RPC_TIMEOUT))
249 +
250 + def close(self):
251 + """Destroy the ZeroMQ Context."""
252 + # Close all sockets and terminate the context.
253 + self.context.destroy(linger=0)
254 +
255 + # Cancel background tasks.
256 + if self.health_loop is not None:
257 + self.health_loop.cancel()
258 + self.output_loop.cancel()
259 +
260 + def _set_errored(self, e: BaseException):
261 + logger.exception(repr(e))
262 + if self._errored_with is None:
263 + self._errored_with = e
264 +
265 + @staticmethod
266 + async def _send_get_data_rpc_request(request: RPCStartupRequest,
267 + expected_type: Any,
268 + error_message: str,
269 + socket: Socket) -> Any:
270 + """Send an RPC request that is expecting data back."""
271 +
272 + # Ping RPCServer with a request.
273 + await socket.send_multipart((pickle.dumps(request), ), copy=False)
274 +
275 + # Make sure the server responds in time.
276 + if await socket.poll(timeout=VLLM_RPC_TIMEOUT) == 0:
277 + raise TimeoutError("RPCServer didn't reply within "
278 + f"{VLLM_RPC_TIMEOUT} ms")
279 +
280 + # Await the data from the Server.
281 + frame = await socket.recv(copy=False)
282 + data = pickle.loads(frame.buffer)
283 +
284 + if isinstance(data, BaseException):
285 + raise data
286 + elif not isinstance(data, expected_type):
287 + raise ValueError(error_message)
288 +
289 + return data
290 +
291 + @staticmethod
292 + async def _send_one_way_rpc_request(request: RPC_REQUEST_T,
293 + socket: Socket):
294 + """Send one-way RPC request to trigger an action."""
295 +
296 + if socket.closed:
297 + raise MQClientClosedError()
298 +
299 + await socket.send_multipart((pickle.dumps(request), ))
300 +
301 + async def _await_ack(self, error_message: str, socket: Socket):
302 + """Await acknowledgement that a request succeeded."""
303 +
304 + if socket.closed:
305 + raise MQClientClosedError()
306 +
307 + if await socket.poll(timeout=VLLM_RPC_TIMEOUT) == 0:
308 + raise TimeoutError("MQLLMEngine didn't reply within "
309 + f"{VLLM_RPC_TIMEOUT}ms")
310 +
311 + await self._check_success(error_message, socket)
312 +
313 + @staticmethod
314 + async def _check_success(error_message: str, socket: Socket):
315 + """Confirm that socket has a VLLM_RPC_SUCCESS_STR message"""
316 +
317 + if socket.closed:
318 + raise MQClientClosedError()
319 +
320 + frame = await socket.recv(copy=False)
321 + response = pickle.loads(frame.buffer)
322 +
323 + # Raise error if unsuccessful
324 + if isinstance(response, BaseException):
325 + raise response
326 + elif (not isinstance(response, str)
327 + or response != VLLM_RPC_SUCCESS_STR):
328 + raise ValueError(error_message)
329 +
330 + async def get_tokenizer(self, lora_request: LoRARequest):
331 + return await self.tokenizer.get_lora_tokenizer_async(lora_request)
332 +
333 + async def get_decoding_config(self) -> DecodingConfig:
334 + return self.decoding_config
335 +
336 + async def get_model_config(self) -> ModelConfig:
337 + return self.model_config
338 +
339 + async def is_tracing_enabled(self) -> bool:
340 + return self.tracing_flag
341 +
342 + async def _wait_for_server_rpc(self, socket: Socket) -> RPCStartupResponse:
343 + """Wait for the RPCServer to start up."""
344 +
345 + return await self._send_get_data_rpc_request(
346 + request=RPCStartupRequest.IS_SERVER_READY,
347 + expected_type=RPCStartupResponse,
348 + error_message="Unable to start RPC Server",
349 + socket=socket)
350 +
351 + async def abort(self, request_id: str):
352 + """Send an ABORT_REQUEST signal to the RPC Server"""
353 +
354 + with suppress(MQClientClosedError):
355 + await self._send_one_way_rpc_request(
356 + request=RPCAbortRequest(request_id), socket=self.input_socket)
357 +
358 + async def do_log_stats(self):
359 + """Ignore do_log_stats (handled on MQLLMEngine polling)"""
360 + pass
361 +
362 + async def check_health(self):
363 + """
364 + The check health loop probes the health status of the
365 + Engine's health every N seconds and sets _errored_with
366 + if the engine is unhealthy.
367 + """
368 + if self._errored_with is not None:
369 + raise self._errored_with
370 +
371 + @property
372 + def is_running(self) -> bool:
373 + return not self.errored
374 +
375 + @property
376 + def is_stopped(self) -> bool:
377 + return self.errored
378 +
379 + @property
380 + def errored(self) -> bool:
381 + return self._errored_with is not None
382 +
383 + async def generate(
384 + self,
385 + inputs: PromptInputs,
386 + sampling_params: SamplingParams,
387 + request_id: str,
388 + lora_request: Optional[LoRARequest] = None,
389 + trace_headers: Optional[Mapping[str, str]] = None,
390 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
391 + ) -> AsyncGenerator[RequestOutput, None]:
392 + """Send an RPCGenerateRequest to the RPCServer and stream responses."""
393 +
394 + # If already dead, error out.
395 + if self._errored_with is not None:
396 + raise ENGINE_DEAD_ERROR(self._errored_with)
397 +
398 + # 1) Create output queue for this requests.
399 + queue: asyncio.Queue[Union[RequestOutput,
400 + BaseException]] = asyncio.Queue()
401 + self.output_queues[request_id] = queue
402 +
403 + try:
404 + # 2) Detach logits processors so that they can be pickled
405 + # separately (may require cloudpickle which is slower)
406 + if sampling_params.logits_processors:
407 + # Defensive shallow copy
408 + sampling_params = copy.copy(sampling_params)
409 + logits_processors = sampling_params.logits_processors
410 + sampling_params.logits_processors = None
411 + lp_bytes = cloudpickle.dumps(logits_processors)
412 + else:
413 + lp_bytes = None
414 +
415 + request_bytes = pickle.dumps(
416 + RPCGenerateRequest(
417 + inputs=inputs,
418 + sampling_params=sampling_params,
419 + request_id=request_id,
420 + lora_request=lora_request,
421 + trace_headers=trace_headers,
422 + prompt_adapter_request=prompt_adapter_request))
423 +
424 + # 3) Send the RPCGenerateRequest to the MQLLMEngine.
425 + parts = (request_bytes,
426 + lp_bytes) if lp_bytes else (request_bytes, )
427 + await self.input_socket.send_multipart(parts, copy=False)
428 +
429 + # 4) Stream the RequestOutputs from the output queue. Note
430 + # that the output_loop pushes RequestOutput objects to this
431 + # queue after pulling them from the zmq socket.
432 + finished = False
433 + try:
434 + while not finished:
435 + request_output = await queue.get()
436 +
437 + if isinstance(request_output, BaseException):
438 + raise request_output
439 +
440 + finished = request_output.finished
441 + yield request_output
442 + finally:
443 + # Request was canceled by the client.
444 + if not finished and not self.errored:
445 + await self.abort(request_id)
446 + finally:
447 + self.output_queues.pop(request_id)
448 +
449 + async def encode(self, *args,
450 + **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
451 + raise NotImplementedError(
452 + "Embeddings not supported with multiprocessing backend")
vllm/engine/multiprocessing/engine.py new
+321
@@ -0,0 +1,321 @@
1 +import pickle
2 +import signal
3 +from contextlib import contextmanager
4 +from typing import Iterator, List, Optional, Union
5 +
6 +import cloudpickle
7 +import zmq
8 +
9 +from vllm import AsyncEngineArgs, LLMEngine
10 +from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11 + ParallelConfig, SchedulerConfig)
12 +# yapf conflicts with isort for this block
13 +# yapf: disable
14 +from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
15 + IPC_HEALTH_EXT, IPC_INPUT_EXT,
16 + IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
17 + VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
18 + RPCError, RPCGenerateRequest,
19 + RPCHealthRequest, RPCStartupRequest,
20 + RPCStartupResponse)
21 +# yapf: enable
22 +from vllm.logger import init_logger
23 +from vllm.outputs import RequestOutput
24 +from vllm.usage.usage_lib import UsageContext
25 +
26 +CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
27 + SchedulerConfig, LoRAConfig]
28 +
29 +logger = init_logger(__name__)
30 +
31 +POLLING_TIMEOUT_MS = 10000
32 +HEALTHY_RESPONSE = (pickle.dumps(VLLM_RPC_SUCCESS_STR), )
33 +
34 +
35 +class MQLLMEngine:
36 + """A multiprocessing wrapper for :class:`LLMEngine`.
37 +
38 + This class is used to wrap the :class:`LLMEngine` class to enable use
39 + in concurrnet manner. It runs a background loop and uses zeromq to
40 + receive new requests and stream outputs incrementally via ipc.
41 +
42 + The :class:`LLMEngine.generate` is kicked off when a new
43 + RPCGenerateRequest is received by the input_socket.
44 +
45 + The self.engine_loop checks the input_socket for new requests,
46 + adds them to the LLMEngine if there are any, calls the internal
47 + :class:`LLMEngine.step()`, and sends the RequestOutputs back over
48 + the output_socket.
49 +
50 + If use_async_sockets is set, the logic associated with reading new
51 + requests from the socket and sending data to the socket is passed
52 + as a callback to the llm_engine, which calls the logic asynchronously
53 + such that the IPC can be overlapped with the GPU.
54 +
55 + Args:
56 + ipc_path: Base path for zeromq interprocess messaging
57 + use_async_sockets: Whether to make send/recv async with GPU
58 + log_requests: Whether to log the requests.
59 + *args: Arguments for :class:`LLMEngine`.
60 + **kwargs: Arguments for :class:`LLMEngine`.
61 + """
62 +
63 + def __init__(self,
64 + ipc_path: str,
65 + use_async_sockets: bool,
66 + *args,
67 + log_requests: bool = True,
68 + **kwargs) -> None:
69 + self.engine = LLMEngine(*args, **kwargs)
70 + self.log_requests = log_requests
71 +
72 + self.use_async_sockets = use_async_sockets
73 + if self.use_async_sockets:
74 + self.engine.process_request_outputs_callback = \
75 + self._async_socket_engine_callback
76 +
77 + self.ctx = zmq.Context() # type: ignore[attr-defined]
78 +
79 + # Receive input from the client.
80 + self.input_socket = self.ctx.socket(zmq.constants.PULL)
81 + self.input_socket.bind(f"{ipc_path}{IPC_INPUT_EXT}")
82 +
83 + # Send output stream back to client.
84 + self.output_socket = self.ctx.socket(zmq.constants.PUSH)
85 + self.output_socket.bind(f"{ipc_path}{IPC_OUTPUT_EXT}")
86 +
87 + # Send health status back to client.
88 + self.health_socket = self.ctx.socket(zmq.constants.PUSH)
89 + self.health_socket.bind(f"{ipc_path}{IPC_HEALTH_EXT}")
90 +
91 + # IPC path for the data socket.
92 + self.data_ipc_path = f"{ipc_path}{IPC_DATA_EXT}"
93 +
94 + # Error state.
95 + self._errored_with: Optional[BaseException] = None
96 +
97 + @property
98 + def dead_error(self) -> BaseException:
99 + if self._errored_with is not None:
100 + return ENGINE_DEAD_ERROR(self._errored_with)
101 + else:
102 + return ENGINE_DEAD_ERROR()
103 +
104 + @classmethod
105 + def from_engine_args(cls, engine_args: AsyncEngineArgs,
106 + usage_context: UsageContext, ipc_path: str):
107 + """Creates an MQLLMEngine from the engine arguments."""
108 +
109 + engine_config = engine_args.create_engine_config()
110 +
111 + executor_class = LLMEngine._get_executor_cls(engine_config)
112 +
113 + return cls(
114 + ipc_path=ipc_path,
115 + use_async_sockets=engine_config.model_config.use_async_output_proc,
116 + **engine_config.to_dict(),
117 + executor_class=executor_class,
118 + log_requests=not engine_args.disable_log_requests,
119 + log_stats=not engine_args.disable_log_stats,
120 + usage_context=usage_context)
121 +
122 + def start(self):
123 + try:
124 + try:
125 + logger.debug("Starting Startup Loop.")
126 + self.run_startup_loop()
127 + logger.debug("Starting Engine Loop.")
128 + self.run_engine_loop()
129 + except Exception as e:
130 + logger.exception(repr(e))
131 + except KeyboardInterrupt:
132 + logger.debug("Shutting down MQLLMEngine.")
133 + finally:
134 + logger.debug("MQLLMEngine is shut down.")
135 + self.cleanup()
136 +
137 + def cleanup(self):
138 + """Cleanup zeromq state on shutdown."""
139 + # Closes all sockets and destroys context.
140 + self.ctx.destroy(linger=0)
141 + del self.engine
142 +
143 + @contextmanager
144 + def make_data_socket(
145 + self) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
146 + socket = self.ctx.socket(zmq.constants.ROUTER)
147 + try:
148 + socket.bind(self.data_ipc_path)
149 + yield socket
150 + finally:
151 + socket.close(linger=0)
152 +
153 + def run_startup_loop(self) -> None:
154 + """Startup loop for sending data from Engine -> Client."""
155 +
156 + with self.make_data_socket() as socket:
157 + response: Union[RPCStartupResponse, BaseException]
158 + try:
159 + identity, message = socket.recv_multipart(copy=False)
160 + request: RPCStartupRequest = pickle.loads(message.buffer)
161 +
162 + # Handle the query from the Client.
163 + if request == RPCStartupRequest.IS_SERVER_READY:
164 + tracing_enabled = self.engine.is_tracing_enabled()
165 + response = RPCStartupResponse(
166 + tracing_enabled=tracing_enabled)
167 +
168 + except Exception as e:
169 + response = e
170 +
171 + socket.send_multipart((identity, pickle.dumps(response)),
172 + copy=False)
173 +
174 + def run_engine_loop(self):
175 + """Core busy loop of the LLMEngine."""
176 +
177 + while True:
178 + if not self.engine.has_unfinished_requests():
179 + # Poll until there is work to do.
180 + while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
181 + self.engine.do_log_stats()
182 + logger.debug("Waiting for new requests in engine loop.")
183 +
184 + # Handle any input from the client.
185 + self.handle_new_input()
186 +
187 + # Engine step.
188 + request_outputs = self.engine_step()
189 +
190 + # Send request outputs (if async, done in engine_step callback).
191 + if not self.use_async_sockets:
192 + self._send_outputs(request_outputs)
193 +
194 + def engine_step(self) -> List[RequestOutput]:
195 + """Engine step wrapper with error handling."""
196 +
197 + try:
198 + return self.engine.step()
199 + except SystemExit:
200 + raise
201 + except BaseException as e:
202 + self._set_errored(e)
203 + rpc_err = RPCError(request_id=None,
204 + is_engine_errored=True,
205 + exception=e)
206 + self._send_outputs(rpc_err)
207 + raise e
208 +
209 + def handle_new_input(self):
210 + """Handle new input from the socket"""
211 + try:
212 + while self.input_socket.poll(timeout=0) != 0:
213 + frames = self.input_socket.recv_multipart(copy=False)
214 + request = pickle.loads(frames[0].buffer)
215 +
216 + if isinstance(request, RPCGenerateRequest):
217 + if len(frames) > 1:
218 + # Use cloudpickle for logits processors
219 + lprocs = cloudpickle.loads(frames[1].buffer)
220 + request.sampling_params.logits_processors = lprocs
221 + self._handle_generate_request(request)
222 + elif isinstance(request, RPCAbortRequest):
223 + self._handle_abort_request(request)
224 + elif isinstance(request, RPCHealthRequest):
225 + self._handle_health_request()
226 + else:
227 + raise ValueError("Unknown RPCRequest Type: {request}")
228 +
229 + except Exception as e:
230 + self._set_errored(e)
231 + self._send_unhealthy(e)
232 + raise e
233 +
234 + def _handle_generate_request(self, request: RPCGenerateRequest):
235 + """Handle RPCGenerateRequest by adding it to the LLMEngine."""
236 + request_id = request.request_id
237 +
238 + if self._errored_with is not None:
239 + rpc_err = RPCError(request_id=request_id,
240 + is_engine_errored=True,
241 + exception=ENGINE_DEAD_ERROR(self._errored_with))
242 + self._send_outputs(rpc_err)
243 +
244 + try:
245 + self.engine.add_request(
246 + request_id=request_id,
247 + inputs=request.inputs,
248 + params=request.sampling_params,
249 + lora_request=request.lora_request,
250 + trace_headers=request.trace_headers,
251 + prompt_adapter_request=request.prompt_adapter_request)
252 +
253 + if self.log_requests:
254 + logger.info("Added request %s.", request.request_id)
255 +
256 + except Exception as e:
257 + # We do not set self._errored = True here, since the error
258 + # is due to an issue adding this request to the engine,
259 + # rather than an issue with the engine itself.
260 + is_errored = self._errored_with is not None
261 + rpc_err = RPCError(request_id=request_id,
262 + is_engine_errored=is_errored,
263 + exception=e)
264 + self._send_outputs(rpc_err)
265 +
266 + # Remove request from the engine.
267 + self.engine.abort_request(request_id)
268 +
269 + def _handle_abort_request(self, request: RPCAbortRequest):
270 + self.engine.abort_request(request.request_id)
271 + if self.log_requests:
272 + logger.info("Aborted request %s.", request.request_id)
273 +
274 + def _handle_health_request(self):
275 + if self._errored_with is not None:
276 + self._send_unhealthy(self._errored_with)
277 +
278 + # Raises error if unhealthy.
279 + self.engine.check_health()
280 + self._send_healthy()
281 +
282 + def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
283 + """Send List of RequestOutput to RPCClient."""
284 + if outputs:
285 + output_bytes = pickle.dumps(outputs)
286 + self.output_socket.send_multipart((output_bytes, ), copy=False)
287 +
288 + def _send_healthy(self):
289 + """Send HEALTHY message to RPCClient."""
290 + self.health_socket.send_multipart(HEALTHY_RESPONSE, copy=False)
291 +
292 + def _send_unhealthy(self, error: BaseException):
293 + """Send UNHEALTHY message to RPCClient."""
294 + error_bytes = pickle.dumps(error)
295 + self.health_socket.send_multipart((error_bytes, ), copy=False)
296 +
297 + def _async_socket_engine_callback(self,
298 + request_outputs: REQUEST_OUTPUTS_T):
299 + """Callback used by engine to make socket handling async with GPU."""
300 + self._send_outputs(request_outputs)
301 + self.handle_new_input()
302 +
303 + def _set_errored(self, e: BaseException):
304 + """Log and set errored status if this is the first issue."""
305 + if self._errored_with is None:
306 + self._errored_with = e
307 +
308 +
309 +def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
310 + ipc_path: str):
311 +
312 + def signal_handler(*_) -> None:
313 + # Interrupt server on sigterm
314 + raise KeyboardInterrupt("MQLLMEngine terminated")
315 +
316 + signal.signal(signal.SIGTERM, signal_handler)
317 +
318 + engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
319 + usage_context=usage_context,
320 + ipc_path=ipc_path)
321 + engine.start()
vllm/engine/protocol.py
+4 -4
@@ -14,8 +14,8 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
14
15
16 @runtime_checkable
17 -class AsyncEngineClient(Protocol):
18 - """Protocol class for Clients to AsyncLLMEngine"""
17 +class EngineClient(Protocol):
18 + """Protocol class for Clients to Engine"""
19
20 @property
21 def is_running(self) -> bool:
@@ -30,8 +30,8 @@ class AsyncEngineClient(Protocol):
30 ...
31
32 @property
33 - def limit_concurrency(self) -> Optional[int]:
34 - """Maximum number of concurrently running requests."""
33 + def dead_error(self) -> BaseException:
34 + ...
35
36 def generate(
37 self,
vllm/entrypoints/launcher.py
+16 -14
@@ -1,21 +1,21 @@
1 import asyncio
2 import signal
3 from http import HTTPStatus
4 -from typing import Any, Optional
4 +from typing import Any
5
6 import uvicorn
7 from fastapi import FastAPI, Request, Response
8
9 from vllm import envs
10 from vllm.engine.async_llm_engine import AsyncEngineDeadError
11 +from vllm.engine.multiprocessing import MQEngineDeadError
12 from vllm.logger import init_logger
13 from vllm.utils import find_process_using_port
14
15 logger = init_logger(__name__)
16
17
17 -async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
18 - **uvicorn_kwargs: Any):
18 +async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
19 logger.info("Available routes are:")
20 for route in app.routes:
21 methods = getattr(route, "methods", None)
@@ -26,15 +26,6 @@ async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
26
27 logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
28
29 - # Set concurrency limits in uvicorn if running in multiprocessing mode
30 - # since zmq has maximum socket limit of zmq.constants.SOCKET_LIMIT (65536).
31 - if limit_concurrency is not None:
32 - logger.info(
33 - "Launching Uvicorn with --limit_concurrency %s. To avoid this "
34 - "limit at the expense of performance run with "
35 - "--disable-frontend-multiprocessing", limit_concurrency)
36 - uvicorn_kwargs["limit_concurrency"] = limit_concurrency
37 -
29 config = uvicorn.Config(app, **uvicorn_kwargs)
30 server = uvicorn.Server(config)
31 _add_shutdown_handlers(app, server)
@@ -63,7 +54,7 @@ async def serve_http(app: FastAPI, limit_concurrency: Optional[int],
54 logger.debug(
55 "port %s is used by process %s launched with command:\n%s",
56 port, process, " ".join(process.cmdline()))
66 - logger.info("Gracefully stopping http server")
57 + logger.info("Shutting down FastAPI HTTP server.")
58 return server.shutdown()
59
60
@@ -90,7 +81,7 @@ def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
81 return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
82
83 @app.exception_handler(AsyncEngineDeadError)
93 - async def engine_dead_handler(_, __):
84 + async def async_engine_dead_handler(_, __):
85 """Kill the server if the async engine is already dead. It will
86 not handle any further requests."""
87 if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
@@ -99,3 +90,14 @@ def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
90 server.should_exit = True
91
92 return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
93 +
94 + @app.exception_handler(MQEngineDeadError)
95 + async def mq_engine_dead_handler(_, __):
96 + """Kill the server if the mq engine is already dead. It will
97 + not handle any further requests."""
98 + if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH:
99 + logger.fatal("MQLLMEngine is already dead, terminating server "
100 + "process")
101 + server.should_exit = True
102 +
103 + return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
vllm/entrypoints/openai/api_server.py
+55 -66
@@ -26,7 +26,9 @@ import vllm.envs as envs
26 from vllm.config import ModelConfig
27 from vllm.engine.arg_utils import AsyncEngineArgs
28 from vllm.engine.async_llm_engine import AsyncLLMEngine
29 -from vllm.engine.protocol import AsyncEngineClient
29 +from vllm.engine.multiprocessing.client import MQLLMEngineClient
30 +from vllm.engine.multiprocessing.engine import run_mp_engine
31 +from vllm.engine.protocol import EngineClient
32 from vllm.entrypoints.launcher import serve_http
33 from vllm.entrypoints.logger import RequestLogger
34 from vllm.entrypoints.openai.cli_args import make_arg_parser
@@ -44,8 +46,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
46 TokenizeRequest,
47 TokenizeResponse,
48 UnloadLoraAdapterRequest)
47 -from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
48 -from vllm.entrypoints.openai.rpc.server import run_rpc_server
49 # yapf: enable
50 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
51 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -67,29 +67,16 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
67 _running_tasks: Set[asyncio.Task] = set()
68
69
70 -def model_is_embedding(model_name: str, trust_remote_code: bool,
71 - quantization: Optional[str],
72 - revision: Optional[str]) -> bool:
73 - return ModelConfig(model=model_name,
74 - revision=revision,
75 - tokenizer=model_name,
76 - tokenizer_mode="auto",
77 - trust_remote_code=trust_remote_code,
78 - quantization=quantization,
79 - seed=0,
80 - dtype="auto").embedding_mode
81 -
82 -
70 @asynccontextmanager
71 async def lifespan(app: FastAPI):
72 try:
73 if app.state.log_stats:
87 - async_engine_client = app.state.engine_client
74 + engine_client: EngineClient = app.state.engine_client
75
76 async def _force_log():
77 while True:
91 - await asyncio.sleep(10)
92 - await async_engine_client.do_log_stats()
78 + await asyncio.sleep(10.)
79 + await engine_client.do_log_stats()
80
81 task = asyncio.create_task(_force_log())
82 _running_tasks.add(task)
@@ -108,9 +95,9 @@ async def lifespan(app: FastAPI):
95
96 @asynccontextmanager
97 async def build_async_engine_client(
111 - args: Namespace) -> AsyncIterator[Optional[AsyncEngineClient]]:
98 + args: Namespace) -> AsyncIterator[Optional[EngineClient]]:
99
113 - # Context manager to handle async_engine_client lifecycle
100 + # Context manager to handle engine_client lifecycle
101 # Ensures everything is shutdown and cleaned up on error/exit
102 engine_args = AsyncEngineArgs.from_cli_args(args)
103
@@ -123,19 +110,18 @@ async def build_async_engine_client(
110 async def build_async_engine_client_from_engine_args(
111 engine_args: AsyncEngineArgs,
112 disable_frontend_multiprocessing: bool = False,
126 -) -> AsyncIterator[Optional[AsyncEngineClient]]:
113 +) -> AsyncIterator[Optional[EngineClient]]:
114 """
128 - Create AsyncEngineClient, either:
115 + Create EngineClient, either:
116 - in-process using the AsyncLLMEngine Directly
117 - multiprocess using AsyncLLMEngine RPC
118
119 Returns the Client or None if the creation failed.
120 """
121
135 - # If manually triggered or embedding model, use AsyncLLMEngine in process.
136 - # TODO: support embedding model via RPC.
137 - if (model_is_embedding(engine_args.model, engine_args.trust_remote_code,
138 - engine_args.quantization, engine_args.revision)
122 + # Fall back
123 + # TODO: fill out feature matrix.
124 + if (MQLLMEngineClient.is_unsupported_config(engine_args)
125 or disable_frontend_multiprocessing):
126 engine_config = engine_args.create_engine_config()
127 uses_ray = getattr(AsyncLLMEngine._get_executor_cls(engine_config),
@@ -173,56 +159,60 @@ async def build_async_engine_client_from_engine_args(
159 "and vLLM will properly handle cleanup.")
160
161 # Select random path for IPC.
176 - rpc_path = get_open_zmq_ipc_path()
177 - logger.info("Multiprocessing frontend to use %s for RPC Path.",
178 - rpc_path)
179 -
180 - # Build RPCClient, which conforms to AsyncEngineClient Protocol.
181 - # NOTE: Actually, this is not true yet. We still need to support
182 - # embedding models via RPC (see TODO above)
183 - rpc_client = AsyncEngineRPCClient(rpc_path)
162 + ipc_path = get_open_zmq_ipc_path()
163 + logger.info("Multiprocessing frontend to use %s for IPC Path.",
164 + ipc_path)
165
185 - # Start RPCServer in separate process (holds the AsyncLLMEngine).
186 - context = multiprocessing.get_context("spawn")
166 + # Start RPCServer in separate process (holds the LLMEngine).
167 # the current process might have CUDA context,
168 # so we need to spawn a new process
189 - rpc_server_process = context.Process(
190 - target=run_rpc_server,
191 - args=(engine_args, UsageContext.OPENAI_API_SERVER, rpc_path))
192 - rpc_server_process.start()
193 - logger.info("Started engine process with PID %d",
194 - rpc_server_process.pid)
169 + context = multiprocessing.get_context("spawn")
170 +
171 + engine_process = context.Process(target=run_mp_engine,
172 + args=(engine_args,
173 + UsageContext.OPENAI_API_SERVER,
174 + ipc_path))
175 + engine_process.start()
176 + logger.info("Started engine process with PID %d", engine_process.pid)
177 +
178 + # Build RPCClient, which conforms to EngineClient Protocol.
179 + # NOTE: Actually, this is not true yet. We still need to support
180 + # embedding models via RPC (see TODO above)
181 + engine_config = engine_args.create_engine_config()
182 + mp_engine_client = MQLLMEngineClient(ipc_path, engine_config)
183
184 try:
185 while True:
186 try:
199 - await rpc_client.setup()
187 + await mp_engine_client.setup()
188 break
189 except TimeoutError:
202 - if not rpc_server_process.is_alive():
203 - logger.error(
204 - "RPCServer process died before responding "
205 - "to readiness probe")
190 + if not engine_process.is_alive():
191 + logger.error("Engine process died before responding "
192 + "to readiness probe")
193 yield None
194 return
195
209 - yield rpc_client # type: ignore[misc]
196 + yield mp_engine_client # type: ignore[misc]
197 finally:
198 # Ensure rpc server process was terminated
212 - rpc_server_process.terminate()
199 + engine_process.terminate()
200
201 # Close all open connections to the backend
215 - rpc_client.close()
202 + mp_engine_client.close()
203
217 - # Wait for server process to join
218 - rpc_server_process.join()
204 + # Wait for engine process to join
205 + engine_process.join(4)
206 + if engine_process.exitcode is None:
207 + # Kill if taking longer than 5 seconds to stop
208 + engine_process.kill()
209
210 # Lazy import for prometheus multiprocessing.
211 # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
212 # before prometheus_client is imported.
213 # See https://prometheus.github.io/client_python/multiprocess/
214 from prometheus_client import multiprocess
225 - multiprocess.mark_process_dead(rpc_server_process.pid)
215 + multiprocess.mark_process_dead(engine_process.pid)
216
217
218 router = APIRouter()
@@ -270,7 +260,7 @@ def embedding(request: Request) -> OpenAIServingEmbedding:
260 return request.app.state.openai_serving_embedding
261
262
273 -def engine_client(request: Request) -> AsyncEngineClient:
263 +def engine_client(request: Request) -> EngineClient:
264 return request.app.state.engine_client
265
266
@@ -473,7 +463,7 @@ def build_app(args: Namespace) -> FastAPI:
463
464
465 def init_app_state(
476 - async_engine_client: AsyncEngineClient,
466 + engine_client: EngineClient,
467 model_config: ModelConfig,
468 state: State,
469 args: Namespace,
@@ -488,11 +478,11 @@ def init_app_state(
478 else:
479 request_logger = RequestLogger(max_log_len=args.max_log_len)
480
491 - state.engine_client = async_engine_client
481 + state.engine_client = engine_client
482 state.log_stats = not args.disable_log_stats
483
484 state.openai_serving_chat = OpenAIServingChat(
495 - async_engine_client,
485 + engine_client,
486 model_config,
487 served_model_names,
488 args.response_role,
@@ -504,7 +494,7 @@ def init_app_state(
494 enable_auto_tools=args.enable_auto_tool_choice,
495 tool_parser=args.tool_call_parser)
496 state.openai_serving_completion = OpenAIServingCompletion(
507 - async_engine_client,
497 + engine_client,
498 model_config,
499 served_model_names,
500 lora_modules=args.lora_modules,
@@ -513,13 +503,13 @@ def init_app_state(
503 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
504 )
505 state.openai_serving_embedding = OpenAIServingEmbedding(
516 - async_engine_client,
506 + engine_client,
507 model_config,
508 served_model_names,
509 request_logger=request_logger,
510 )
511 state.openai_serving_tokenization = OpenAIServingTokenization(
522 - async_engine_client,
512 + engine_client,
513 model_config,
514 served_model_names,
515 lora_modules=args.lora_modules,
@@ -541,21 +531,20 @@ async def run_server(args, **uvicorn_kwargs) -> None:
531
532 signal.signal(signal.SIGTERM, signal_handler)
533
544 - async with build_async_engine_client(args) as async_engine_client:
534 + async with build_async_engine_client(args) as engine_client:
535 # If None, creation of the client failed and we exit.
546 - if async_engine_client is None:
536 + if engine_client is None:
537 return
538
539 app = build_app(args)
540
551 - model_config = await async_engine_client.get_model_config()
552 - init_app_state(async_engine_client, model_config, app.state, args)
541 + model_config = await engine_client.get_model_config()
542 + init_app_state(engine_client, model_config, app.state, args)
543
544 temp_socket.close()
545
546 shutdown_task = await serve_http(
547 app,
558 - limit_concurrency=async_engine_client.limit_concurrency,
548 host=args.host,
549 port=args.port,
550 log_level=args.uvicorn_log_level,
vllm/entrypoints/openai/rpc/__init__.py deleted
-50
@@ -1,50 +0,0 @@
1 -from dataclasses import dataclass
2 -from enum import Enum
3 -from typing import Mapping, Optional, Union
4 -
5 -from vllm.inputs import PromptInputs
6 -from vllm.lora.request import LoRARequest
7 -from vllm.prompt_adapter.request import PromptAdapterRequest
8 -from vllm.sampling_params import SamplingParams
9 -
10 -# Success string used for RPC instructions.
11 -VLLM_RPC_SUCCESS_STR = "SUCCESS"
12 -
13 -# Minimum value of ZMQ.SOCKET_LIMIT to run mp.
14 -VLLM_RPC_SOCKET_LIMIT_CUTOFF = 2000
15 -
16 -# HWM is set to Infinity.
17 -VLLM_RPC_ZMQ_HWM = 0
18 -
19 -
20 -@dataclass
21 -class RPCGenerateRequest:
22 - inputs: PromptInputs
23 - sampling_params: SamplingParams
24 - request_id: str
25 - lora_request: Optional[LoRARequest] = None
26 - trace_headers: Optional[Mapping[str, str]] = None
27 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
28 -
29 -
30 -@dataclass
31 -class RPCAbortRequest:
32 - request_id: str
33 -
34 -
35 -class RPCUtilityRequest(Enum):
36 - IS_SERVER_READY = 1
37 - GET_MODEL_CONFIG = 2
38 - GET_DECODING_CONFIG = 3
39 - GET_PARALLEL_CONFIG = 4
40 - GET_SCHEDULER_CONFIG = 5
41 - GET_LORA_CONFIG = 6
42 - DO_LOG_STATS = 7
43 - IS_SERVER_HEALTHY = 8
44 - IS_TRACING_ENABLED = 9
45 - START_PROFILE = 10
46 - STOP_PROFILE = 11
47 -
48 -
49 -RPC_REQUEST_TYPE = Union[RPCGenerateRequest, RPCAbortRequest,
50 - RPCUtilityRequest]
vllm/entrypoints/openai/rpc/client.py deleted
-451
@@ -1,451 +0,0 @@
1 -import asyncio
2 -import pickle
3 -from contextlib import contextmanager, suppress
4 -from typing import Any, AsyncGenerator, Iterator, Mapping, Optional
5 -from uuid import uuid4
6 -
7 -import cloudpickle
8 -import zmq
9 -import zmq.asyncio
10 -from zmq import Frame # type: ignore[attr-defined]
11 -from zmq.asyncio import Socket
12 -
13 -from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
14 - ParallelConfig, SchedulerConfig)
15 -# yapf: disable
16 -from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
17 - VLLM_RPC_SOCKET_LIMIT_CUTOFF,
18 - VLLM_RPC_SUCCESS_STR,
19 - VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
20 - RPCGenerateRequest, RPCUtilityRequest)
21 -# yapf: enable
22 -from vllm.envs import VLLM_RPC_GET_DATA_TIMEOUT_MS
23 -from vllm.inputs import PromptInputs
24 -from vllm.logger import init_logger
25 -from vllm.lora.request import LoRARequest
26 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27 -from vllm.prompt_adapter.request import PromptAdapterRequest
28 -from vllm.sampling_params import SamplingParams
29 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
30 -
31 -logger = init_logger(__name__)
32 -
33 -# Path used for inprocess proxy.
34 -INPROC_PROXY_PATH = f"inproc://{uuid4()}"
35 -
36 -
37 -class RPCClientClosedError(Exception):
38 - """Exception class raised when the client is used post-close.
39 -
40 - The client can be closed, which closes the ZMQ context. This normally
41 - happens on server shutdown. In some cases, methods like abort and
42 - do_log_stats will still be called and then try to open a socket, which
43 - causes a ZMQError and creates a huge stack trace.
44 - So, we throw this error such that we can suppress it.
45 - """
46 -
47 -
48 -class AsyncEngineRPCClient:
49 - """
50 - RPCClient that connects to the RPCServer wrapping AsyncLLMEngine.
51 -
52 - The overall design mirrors the Asynchronous Client Server Pattern
53 - https://zguide.zeromq.org/docs/chapter3/#The-Asynchronous-Client-Server-Pattern
54 -
55 - On startup, the RPCClient:
56 - - makes DEALER socket (to_rpc_server) that connects to the RPCServer
57 - via ipc, which uses unix sockets under the hood
58 - (https://libzmq.readthedocs.io/en/zeromq4-1/zmq_ipc.html)
59 - - makes ROUTER socket (from_api_server) that binds to a random
60 - inproc address, which uses memory under the hood
61 - (https://libzmq.readthedocs.io/en/zeromq3-x/zmq_inproc.html)
62 - - runs a proxy in a background asyncio task between
63 - from_api_server (ROUTER, inproc) and to_rpc_server (DEALER ipc, )
64 -
65 - Each request handled by the asyncio api_server calls generate():
66 - - make a DEALER socket that connects to from_api_server via inproc
67 - - send a RCPGenerateRequest to the inproc socket
68 - - background proxy forwards the request from inproc -> ipc
69 - - RPCServer responds to the request one token at a time over ipc
70 - - background proxy forwards the response from ipc -> inproc
71 -
72 - The connection looks like this:
73 - DEALER <- inproc -> [ ROUTER | DEALER ] <- ipc -> DEALER
74 -
75 - Message routing is performed via identities that are managed by the
76 - ROUTER socket. ROUTER sockets track every connection it has and
77 - tells the caller about these. The way it tells the caller is to stick
78 - the connection identity in front of each message received. When we
79 - send the message via a ROUTER, we first send an identity frame.
80 - See https://zguide.zeromq.org/docs/chapter3/#The-Extended-Reply-Envelope
81 - for more details on connection identities.
82 -
83 - This proxy design enables us to use a single unix socket, which
84 - improves performance by avoiding syscalls (~5%) and avoids resource limits
85 - such as ulimit, which defaults to 1024 on ubuntu.
86 -
87 - Note: we run set_hwm(0) on each socket, which sets the HWM to inf,
88 - which is required to avoid dropping messages under high load.
89 - This is generally not advisable. However, since we are in control
90 - of both sides of the connection + failure on either side is
91 - catastrophic to the overall system health and memory profiling
92 - suggests limited memory overhead relative to asyncio, we will
93 - proceed for now.
94 -
95 - See https://zguide.zeromq.org/docs/chapter2/#High-Water-Marks
96 - for more details on high water marks.
97 - """
98 -
99 - def __init__(self, rpc_path: str):
100 - self.context = zmq.asyncio.Context()
101 - self._data_timeout = VLLM_RPC_GET_DATA_TIMEOUT_MS
102 - self._errored = False
103 -
104 - # Maximum number of sockets that can be opened (typically 65536).
105 - # ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
106 - socket_limit = self.context.get(zmq.constants.SOCKET_LIMIT)
107 - assert isinstance(socket_limit, int)
108 - if socket_limit < VLLM_RPC_SOCKET_LIMIT_CUTOFF:
109 - raise ValueError(
110 - f"Found zmq.constants.SOCKET_LIMIT={socket_limit}, which caps "
111 - "the number of concurrent requests vLLM can process. Launch "
112 - "vLLM with --disable-frontend-multiprocessing and open a "
113 - "GitHub issue so we can investigate.")
114 -
115 - # We only have 1 ipc connection that uses unix sockets, so
116 - # safe to set MAX_SOCKETS to the zmq SOCKET_LIMIT (i.e. will
117 - # not run into ulimit issues)
118 - self.context.set(zmq.constants.MAX_SOCKETS, socket_limit)
119 -
120 - # IPC connection to RPC Server (uses unix sockets).
121 - self.to_rpc_server: Socket = self.context.socket(zmq.constants.DEALER)
122 - self.to_rpc_server.set_hwm(VLLM_RPC_ZMQ_HWM)
123 - self.to_rpc_server.bind(rpc_path)
124 -
125 - # In process proxy to RPC Server (uses memory-based messaging).
126 - self.from_api_server: Socket = self.context.socket(
127 - zmq.constants.ROUTER)
128 - self.from_api_server.set_hwm(VLLM_RPC_ZMQ_HWM)
129 - self.from_api_server.bind(INPROC_PROXY_PATH)
130 -
131 - # Asyncio background task for the proxy.
132 - self.proxy_in_task = asyncio.create_task(
133 - self.run_proxy(self.from_api_server, self.to_rpc_server))
134 - self.proxy_out_task = asyncio.create_task(
135 - self.run_proxy(self.to_rpc_server, self.from_api_server))
136 -
137 - # Since we open 1 inproc socket per request, we have a hard cap on
138 - # the number of requests that can run in vLLM w. frontend
139 - # mulitprocessing. This value is used uvicorn to launch
140 - # with --limit-concurrency to return 503 when server is overloaded.
141 - # We need 2 sockets per request - 2:
142 - # 1 for generate(), 1 for abort(), do_log_stats(), check_health()
143 - self.limit_concurrency = socket_limit // 2 - 2
144 -
145 - async def run_proxy(self, socket_from: Socket, socket_to: Socket):
146 - """Background task that runs a proxy"""
147 - while True:
148 - frames = await socket_from.recv_multipart(copy=False)
149 - await socket_to.send_multipart(frames, copy=False)
150 -
151 - async def setup(self):
152 - """Setup the client before it starts sending server requests."""
153 -
154 - # Wait until server is ready.
155 - await self._wait_for_server_rpc()
156 -
157 - # Get the configs.
158 - self.model_config = await self._get_model_config_rpc()
159 - self.decoding_config = await self._get_decoding_config_rpc()
160 - self.tracing_flag = await self._is_tracing_enabled_rpc()
161 -
162 - # Create the tokenizer group.
163 - # TODO: refactor OAI server to avoid needing this info.
164 - self.tokenizer = init_tokenizer_from_configs(
165 - model_config=self.model_config,
166 - scheduler_config=(await self._get_scheduler_config_rpc()),
167 - parallel_config=(await self._get_parallel_config_rpc()),
168 - enable_lora=bool(await self._get_lora_config_rpc()),
169 - )
170 -
171 - def close(self):
172 - """Destroy the ZeroMQ Context."""
173 - # Close all sockets associated with this context and
174 - # then terminate the context.
175 - self.from_api_server.close()
176 - self.to_rpc_server.close()
177 - self.context.destroy()
178 -
179 - @contextmanager
180 - def to_proxy_socket(self) -> Iterator[Socket]:
181 - # Connect to the RPCServer via the proxy.
182 -
183 - # Raise a sensible error if the client was already closed.
184 - # This can happen if a server shutdown is triggered but some coroutines
185 - # are still running requests.
186 - # There should not be a race condition with this check because we don't
187 - # yield to the event loop between here and opening the socket.
188 - if self.context.closed:
189 - raise RPCClientClosedError("The ZMQ client has already shut down")
190 -
191 - # Note that we use DEALER to enable asynchronous communication
192 - # to enable streaming.
193 - socket = self.context.socket(zmq.constants.DEALER)
194 - socket.set_hwm(VLLM_RPC_ZMQ_HWM)
195 - try:
196 - socket.connect(INPROC_PROXY_PATH)
197 - yield socket
198 - finally:
199 - socket.close(linger=0)
200 -
201 - async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
202 - expected_type: Any,
203 - error_message: str) -> Any:
204 - """Send an RPC request that is expecting data back."""
205 -
206 - with self.to_proxy_socket() as socket:
207 - # Ping RPCServer with a request.
208 - await socket.send_multipart((cloudpickle.dumps(request), ),
209 - copy=False)
210 -
211 - # Make sure the server responds
212 - if await socket.poll(timeout=self._data_timeout) == 0:
213 - raise TimeoutError("Server didn't reply within "
214 - f"{self._data_timeout} ms")
215 -
216 - # Await the data from the Server.
217 - frame = await socket.recv(copy=False)
218 - assert isinstance(frame, Frame)
219 - data = pickle.loads(frame.buffer)
220 -
221 - if isinstance(data, Exception):
222 - # Re-raise exceptions returned by the server
223 - raise data
224 -
225 - if not isinstance(data, expected_type):
226 - # LoRAConfig can be None.
227 - if expected_type == LoRAConfig and data is None:
228 - pass
229 - elif isinstance(data, Exception):
230 - logger.error(error_message)
231 - raise data
232 - else:
233 - raise ValueError(error_message)
234 -
235 - return data
236 -
237 - async def _send_one_way_rpc_request(self,
238 - request: RPC_REQUEST_TYPE,
239 - error_message: str,
240 - socket: Optional[Socket] = None):
241 - """Send one-way RPC request to trigger an action."""
242 -
243 - async def do_rpc_call(socket: Socket, request: RPC_REQUEST_TYPE):
244 -
245 - await socket.send_multipart((cloudpickle.dumps(request), ))
246 -
247 - if await socket.poll(timeout=self._data_timeout) == 0:
248 - raise TimeoutError("Server didn't reply within "
249 - f"{self._data_timeout} ms")
250 -
251 - frame = await socket.recv(copy=False)
252 - assert isinstance(frame, Frame)
253 - return pickle.loads(frame.buffer)
254 -
255 - # Make a new socket connection.
256 - if socket is None:
257 - with self.to_proxy_socket() as socket:
258 - response = await do_rpc_call(socket, request)
259 -
260 - # Use existing socket connection.
261 - else:
262 - response = await do_rpc_call(socket, request)
263 -
264 - if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
265 - if isinstance(response, Exception):
266 - logger.error(error_message)
267 - raise response
268 - raise ValueError(error_message)
269 -
270 - async def get_tokenizer(self, lora_request: LoRARequest):
271 - return await self.tokenizer.get_lora_tokenizer_async(lora_request)
272 -
273 - async def get_decoding_config(self) -> DecodingConfig:
274 - return self.decoding_config
275 -
276 - async def get_model_config(self) -> ModelConfig:
277 - return self.model_config
278 -
279 - async def is_tracing_enabled(self) -> bool:
280 - return self.tracing_flag
281 -
282 - async def _wait_for_server_rpc(self):
283 - """Wait for the RPCServer to start up."""
284 -
285 - await self._send_one_way_rpc_request(
286 - request=RPCUtilityRequest.IS_SERVER_READY,
287 - error_message="Unable to start RPC Server")
288 -
289 - async def _get_model_config_rpc(self) -> ModelConfig:
290 - """Get the ModelConfig object from the RPC Server"""
291 -
292 - return await self._send_get_data_rpc_request(
293 - RPCUtilityRequest.GET_MODEL_CONFIG,
294 - expected_type=ModelConfig,
295 - error_message="Could not get ModelConfig from RPC Server")
296 -
297 - async def _get_decoding_config_rpc(self) -> DecodingConfig:
298 - """Get DecodingConfig from the RPCServer"""
299 -
300 - return await self._send_get_data_rpc_request(
301 - RPCUtilityRequest.GET_DECODING_CONFIG,
302 - expected_type=DecodingConfig,
303 - error_message="Could not get DecodingConfig from RPC Server")
304 -
305 - async def _get_parallel_config_rpc(self) -> ParallelConfig:
306 - """Get ParallelConfig from the RPCServer"""
307 -
308 - return await self._send_get_data_rpc_request(
309 - RPCUtilityRequest.GET_PARALLEL_CONFIG,
310 - expected_type=ParallelConfig,
311 - error_message="Could not get ParallelConfig from RPC Server")
312 -
313 - async def _get_scheduler_config_rpc(self) -> SchedulerConfig:
314 - """Get SchedulerConfig from the RPCServer"""
315 -
316 - return await self._send_get_data_rpc_request(
317 - RPCUtilityRequest.GET_SCHEDULER_CONFIG,
318 - expected_type=SchedulerConfig,
319 - error_message="Could not get SchedulerConfig from RPC Server")
320 -
321 - async def _get_lora_config_rpc(self) -> LoRAConfig:
322 - """Get LoRAConfig from the RPCServer"""
323 -
324 - return await self._send_get_data_rpc_request(
325 - RPCUtilityRequest.GET_LORA_CONFIG,
326 - expected_type=LoRAConfig,
327 - error_message="Could not get LoRAConfig from RPC Server")
328 -
329 - async def _is_tracing_enabled_rpc(self) -> bool:
330 - """Get is_tracing_enabled flag from the RPCServer"""
331 -
332 - return await self._send_get_data_rpc_request(
333 - RPCUtilityRequest.IS_TRACING_ENABLED,
334 - expected_type=bool,
335 - error_message="Could not get is_tracing_enabled from RPC Server")
336 -
337 - async def abort(self, request_id: str):
338 - """Send an ABORT_REQUEST signal to the RPC Server"""
339 -
340 - # Suppress timeouts as well.
341 - # In cases where the server is busy processing requests and a very
342 - # large volume of abort requests arrive, it is likely that the server
343 - # will not be able to ack all of them in time. We have seen this when
344 - # we abort 20k requests at once while another 2k are processing- many
345 - # of them time out, but we see the server successfully abort all of the
346 - # requests.
347 - # In this case we assume that the server has received or will receive
348 - # these abort requests, and ignore the timeout. This prevents a massive
349 - # wall of `TimeoutError` stack traces.
350 - with suppress(RPCClientClosedError, TimeoutError):
351 - await self._send_one_way_rpc_request(
352 - request=RPCAbortRequest(request_id),
353 - error_message=f"RPCAbortRequest {request_id} failed")
354 -
355 - async def do_log_stats(self):
356 - """Send a DO_LOG_STATS signal to the RPC Server"""
357 - with suppress(RPCClientClosedError):
358 - await self._send_one_way_rpc_request(
359 - request=RPCUtilityRequest.DO_LOG_STATS,
360 - error_message="RPCRequest DO_LOG_STATS failed.")
361 -
362 - @property
363 - def is_running(self) -> bool:
364 - return not self._errored
365 -
366 - @property
367 - def is_stopped(self) -> bool:
368 - return self._errored
369 -
370 - @property
371 - def errored(self) -> bool:
372 - return self._errored
373 -
374 - async def generate(
375 - self,
376 - inputs: PromptInputs,
377 - sampling_params: SamplingParams,
378 - request_id: str,
379 - lora_request: Optional[LoRARequest] = None,
380 - trace_headers: Optional[Mapping[str, str]] = None,
381 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
382 - ) -> AsyncGenerator[RequestOutput, None]:
383 - """Send an RPCGenerateRequest to the RPCServer and stream responses."""
384 -
385 - finished = False
386 - try:
387 - with self.to_proxy_socket() as socket:
388 - # Send RPCGenerateRequest to the RPCServer.
389 - await socket.send_multipart((cloudpickle.dumps(
390 - RPCGenerateRequest(
391 - inputs=inputs,
392 - sampling_params=sampling_params,
393 - request_id=request_id,
394 - lora_request=lora_request,
395 - trace_headers=trace_headers,
396 - prompt_adapter_request=prompt_adapter_request)), ))
397 -
398 - # Stream back the results from the RPC Server.
399 - while not finished:
400 - message = await socket.recv(copy=False)
401 - assert isinstance(message, Frame)
402 - request_output = pickle.loads(message.buffer)
403 -
404 - if isinstance(request_output, Exception):
405 - # On exception, check if the server is still healthy
406 - # possibly setting the `errored` property.
407 - if not self._errored:
408 - try:
409 - await self.check_health(socket=socket)
410 - except Exception as e:
411 - self._errored = True
412 - logger.exception(repr(e))
413 -
414 - # NB: do before raising here so that the flag is set
415 - # by the time the caller receives this exception
416 - raise request_output
417 -
418 - finished = request_output.finished
419 - yield request_output
420 -
421 - finally:
422 - # Request was canceled by the client.
423 - if not finished and not self._errored:
424 - await self.abort(request_id)
425 -
426 - async def check_health(self, socket: Optional[Socket] = None) -> None:
427 - """Raise if unhealthy"""
428 -
429 - await self._send_one_way_rpc_request(
430 - request=RPCUtilityRequest.IS_SERVER_HEALTHY,
431 - error_message="Got Unhealthy response from RPC Server",
432 - socket=socket)
433 -
434 - async def encode(self, *args,
435 - **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
436 - raise NotImplementedError(
437 - "Embeddings not supported with multiprocessing backend")
438 -
439 - async def start_profile(self) -> None:
440 - """Start profiling the engine"""
441 -
442 - await self._send_one_way_rpc_request(
443 - request=RPCUtilityRequest.START_PROFILE,
444 - error_message="RPCRequest START_PROFILE failed.")
445 -
446 - async def stop_profile(self) -> None:
447 - """Stop profiling the engine"""
448 -
449 - await self._send_one_way_rpc_request(
450 - request=RPCUtilityRequest.STOP_PROFILE,
451 - error_message="RPCRequest STOP_PROFILE failed.")
vllm/entrypoints/openai/rpc/server.py deleted
-243
@@ -1,243 +0,0 @@
1 -import asyncio
2 -import pickle
3 -import signal
4 -from typing import Any, Coroutine, Union
5 -
6 -import cloudpickle
7 -import uvloop
8 -import zmq
9 -import zmq.asyncio
10 -from typing_extensions import Never
11 -from zmq import Frame # type: ignore[attr-defined]
12 -from zmq.asyncio import Socket
13 -
14 -from vllm import AsyncEngineArgs, AsyncLLMEngine
15 -from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
16 - ParallelConfig, SchedulerConfig)
17 -from vllm.entrypoints.openai.rpc import (VLLM_RPC_SUCCESS_STR,
18 - VLLM_RPC_ZMQ_HWM, RPCAbortRequest,
19 - RPCGenerateRequest, RPCUtilityRequest)
20 -from vllm.logger import init_logger
21 -from vllm.usage.usage_lib import UsageContext
22 -
23 -logger = init_logger(__name__)
24 -
25 -CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
26 - SchedulerConfig, LoRAConfig]
27 -
28 -
29 -class AsyncEngineRPCServer:
30 -
31 - def __init__(self, async_engine_args: AsyncEngineArgs,
32 - usage_context: UsageContext, rpc_path: str):
33 - # Initialize engine first.
34 - self.engine = AsyncLLMEngine.from_engine_args(
35 - async_engine_args, usage_context=usage_context)
36 -
37 - # Initialize context.
38 - self.context = zmq.asyncio.Context()
39 -
40 - # Init socket.
41 - self.socket: Socket = self.context.socket(zmq.constants.DEALER)
42 - self.socket.set_hwm(VLLM_RPC_ZMQ_HWM)
43 - self.socket.connect(rpc_path)
44 -
45 - def cleanup(self):
46 - """Cleanup all resources."""
47 - self.socket.close()
48 - self.context.destroy()
49 - # Clear the engine reference so that it can be GC'ed.
50 - del self.engine
51 -
52 - async def get_config(self, identity, request):
53 - try:
54 - config: CONFIG_TYPE
55 - if request == RPCUtilityRequest.GET_MODEL_CONFIG:
56 - config = await self.engine.get_model_config()
57 - elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
58 - config = await self.engine.get_decoding_config()
59 - elif request == RPCUtilityRequest.GET_LORA_CONFIG:
60 - config = await self.engine.get_lora_config()
61 - elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
62 - config = await self.engine.get_scheduler_config()
63 - elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
64 - config = await self.engine.get_parallel_config()
65 - else:
66 - raise ValueError("Unknown Config Request: %s", request)
67 -
68 - await self.socket.send_multipart((identity, pickle.dumps(config)),
69 - copy=False)
70 -
71 - except Exception as e:
72 - await self.socket.send_multipart((identity, pickle.dumps(e)),
73 - copy=False)
74 -
75 - async def is_tracing_enabled(self, identity):
76 - """Send the is_tracing_enabled flag"""
77 - tracing_flag = await self.engine.is_tracing_enabled()
78 -
79 - await self.socket.send_multipart(
80 - (identity, pickle.dumps(tracing_flag)))
81 -
82 - async def do_log_stats(self, identity):
83 - """Log stats and confirm success."""
84 - await self.engine.do_log_stats()
85 -
86 - await self.socket.send_multipart(
87 - (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
88 -
89 - async def is_server_ready(self, identity):
90 - """Notify the client that we are ready."""
91 - await self.socket.send_multipart(
92 - (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
93 -
94 - async def abort(self, identity, request: RPCAbortRequest):
95 - """Abort request and notify the client of success."""
96 - try:
97 - # Abort the request in the llm engine.
98 - await self.engine.abort(request.request_id)
99 - result: Union[str, Exception] = VLLM_RPC_SUCCESS_STR
100 - except Exception as e:
101 - result = e
102 - await self.socket.send_multipart((identity, pickle.dumps(result)))
103 -
104 - async def generate(self, identity, generate_request: RPCGenerateRequest):
105 - try:
106 - results_generator = self.engine.generate(
107 - generate_request.inputs,
108 - sampling_params=generate_request.sampling_params,
109 - request_id=generate_request.request_id,
110 - lora_request=generate_request.lora_request,
111 - trace_headers=generate_request.trace_headers,
112 - prompt_adapter_request=generate_request.prompt_adapter_request)
113 -
114 - async for request_output in results_generator:
115 - await self.socket.send_multipart(
116 - (identity, pickle.dumps(request_output)), copy=False)
117 -
118 - except Exception as e:
119 - await self.socket.send_multipart((identity, pickle.dumps(e)),
120 - copy=False)
121 -
122 - async def check_health(self, identity):
123 - try:
124 - await self.engine.check_health()
125 - await self.socket.send_multipart(
126 - (identity, pickle.dumps(VLLM_RPC_SUCCESS_STR)))
127 -
128 - except Exception as e:
129 - await self.socket.send_multipart((identity, pickle.dumps(e)),
130 - copy=False)
131 -
132 - async def start_profile(self, identity):
133 - logger.info("Starting profiler...")
134 - await self.engine.start_profile()
135 - logger.info("Profiler started.")
136 -
137 - await self.socket.send_multipart((
138 - identity,
139 - pickle.dumps(VLLM_RPC_SUCCESS_STR),
140 - ))
141 -
142 - async def stop_profile(self, identity):
143 - logger.info("Stopping profiler...")
144 - await self.engine.stop_profile()
145 - logger.info("Profiler stopped.")
146 -
147 - await self.socket.send_multipart((
148 - identity,
149 - pickle.dumps(VLLM_RPC_SUCCESS_STR),
150 - ))
151 -
152 - def _make_handler_coro(self, identity,
153 - message: Frame) -> Coroutine[Any, Any, Never]:
154 - """Route the zmq message to the handler coroutine."""
155 -
156 - request = cloudpickle.loads(message.buffer)
157 -
158 - if isinstance(request, RPCGenerateRequest):
159 - return self.generate(identity, request)
160 -
161 - elif isinstance(request, RPCAbortRequest):
162 - return self.abort(identity, request)
163 -
164 - elif isinstance(request, RPCUtilityRequest):
165 - if request in [
166 - RPCUtilityRequest.GET_MODEL_CONFIG,
167 - RPCUtilityRequest.GET_PARALLEL_CONFIG,
168 - RPCUtilityRequest.GET_DECODING_CONFIG,
169 - RPCUtilityRequest.GET_SCHEDULER_CONFIG,
170 - RPCUtilityRequest.GET_LORA_CONFIG
171 - ]:
172 - return self.get_config(identity, request)
173 - elif request == RPCUtilityRequest.DO_LOG_STATS:
174 - return self.do_log_stats(identity)
175 - elif request == RPCUtilityRequest.IS_SERVER_READY:
176 - return self.is_server_ready(identity)
177 - elif request == RPCUtilityRequest.IS_SERVER_HEALTHY:
178 - return self.check_health(identity)
179 - elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
180 - return self.is_tracing_enabled(identity)
181 - elif request == RPCUtilityRequest.START_PROFILE:
182 - return self.start_profile(identity)
183 - elif request == RPCUtilityRequest.STOP_PROFILE:
184 - return self.stop_profile(identity)
185 - else:
186 - raise ValueError(f"Unknown RPCUtilityRequest type: {request}")
187 -
188 - else:
189 - raise ValueError(f"Unknown RPCRequest type: {request}")
190 -
191 - async def run_server_loop(self):
192 - """Inner RPC Server Loop"""
193 -
194 - running_tasks = set()
195 - while True:
196 - # Wait for a request.
197 - identity, message = await self.socket.recv_multipart(copy=False)
198 -
199 - # Process the request async.
200 - task = asyncio.create_task(
201 - self._make_handler_coro(identity, message))
202 -
203 - # We need to keep around a strong reference to the task,
204 - # to avoid the task disappearing mid-execution as running tasks
205 - # can be GC'ed. Below is a common "fire-and-forget" tasks
206 - # https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
207 - running_tasks.add(task)
208 - task.add_done_callback(running_tasks.discard)
209 -
210 -
211 -async def run_server(server: AsyncEngineRPCServer):
212 - # Put the server task into the asyncio loop.
213 - loop = asyncio.get_running_loop()
214 - server_task = loop.create_task(server.run_server_loop())
215 -
216 - # Interruption handling.
217 - def signal_handler() -> None:
218 - # Kill the server on interrupt / terminate
219 - server_task.cancel()
220 -
221 - loop.add_signal_handler(signal.SIGINT, signal_handler)
222 - loop.add_signal_handler(signal.SIGTERM, signal_handler)
223 -
224 - try:
225 - await server_task
226 - except asyncio.CancelledError:
227 - logger.info("vLLM ZMQ RPC Server was interrupted.")
228 - finally:
229 - # Clean up all resources.
230 - server.cleanup()
231 -
232 -
233 -def run_rpc_server(async_engine_args: AsyncEngineArgs,
234 - usage_context: UsageContext, rpc_path: str):
235 -
236 - def signal_handler(*_) -> None:
237 - # Interrupt server on sigterm while initializing
238 - raise KeyboardInterrupt("AsyncEngineRPCServer terminated")
239 -
240 - signal.signal(signal.SIGTERM, signal_handler)
241 -
242 - server = AsyncEngineRPCServer(async_engine_args, usage_context, rpc_path)
243 - uvloop.run(run_server(server))
vllm/entrypoints/openai/serving_chat.py
+13 -8
@@ -9,7 +9,7 @@ from typing import Union
9 from fastapi import Request
10
11 from vllm.config import ModelConfig
12 -from vllm.engine.protocol import AsyncEngineClient
12 +from vllm.engine.protocol import EngineClient
13 from vllm.entrypoints.chat_utils import (ConversationMessage,
14 apply_hf_chat_template,
15 apply_mistral_chat_template,
@@ -45,7 +45,7 @@ logger = init_logger(__name__)
45 class OpenAIServingChat(OpenAIServing):
46
47 def __init__(self,
48 - async_engine_client: AsyncEngineClient,
48 + engine_client: EngineClient,
49 model_config: ModelConfig,
50 served_model_names: List[str],
51 response_role: str,
@@ -57,7 +57,7 @@ class OpenAIServingChat(OpenAIServing):
57 return_tokens_as_token_ids: bool = False,
58 enable_auto_tools: bool = False,
59 tool_parser: Optional[str] = None):
60 - super().__init__(async_engine_client=async_engine_client,
60 + super().__init__(engine_client=engine_client,
61 model_config=model_config,
62 served_model_names=served_model_names,
63 lora_modules=lora_modules,
@@ -105,6 +105,12 @@ class OpenAIServingChat(OpenAIServing):
105 logger.error("Error with model %s", error_check_ret)
106 return error_check_ret
107
108 + # If the engine is dead, raise the engine's DEAD_ERROR.
109 + # This is required for the streaming case, where we return a
110 + # success status before we actually start generating text :).
111 + if self.engine_client.errored:
112 + raise self.engine_client.dead_error
113 +
114 try:
115 (
116 lora_request,
@@ -112,8 +118,7 @@ class OpenAIServingChat(OpenAIServing):
118 ) = self._maybe_get_adapters(request)
119
120 model_config = self.model_config
115 - tokenizer = await self.async_engine_client.get_tokenizer(
116 - lora_request)
121 + tokenizer = await self.engine_client.get_tokenizer(lora_request)
122
123 conversation, mm_data_future = parse_chat_messages_futures(
124 request.messages, model_config, tokenizer)
@@ -207,8 +212,8 @@ class OpenAIServingChat(OpenAIServing):
212 if mm_data is not None:
213 engine_inputs["multi_modal_data"] = mm_data
214
210 - is_tracing_enabled = (
211 - await self.async_engine_client.is_tracing_enabled())
215 + is_tracing_enabled = (await
216 + self.engine_client.is_tracing_enabled())
217 trace_headers = None
218 if is_tracing_enabled and raw_request:
219 trace_headers = extract_trace_headers(raw_request.headers)
@@ -216,7 +221,7 @@ class OpenAIServingChat(OpenAIServing):
221 and contains_trace_headers(raw_request.headers)):
222 log_tracing_disabled_warning()
223
219 - result_generator = self.async_engine_client.generate(
224 + result_generator = self.engine_client.generate(
225 engine_inputs,
226 sampling_params,
227 request_id,
vllm/entrypoints/openai/serving_completion.py
+13 -8
@@ -8,7 +8,7 @@ from typing import Tuple, Union, cast
8 from fastapi import Request
9
10 from vllm.config import ModelConfig
11 -from vllm.engine.protocol import AsyncEngineClient
11 +from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.logger import RequestLogger
13 # yapf conflicts with isort for this block
14 # yapf: disable
@@ -43,7 +43,7 @@ class OpenAIServingCompletion(OpenAIServing):
43
44 def __init__(
45 self,
46 - async_engine_client: AsyncEngineClient,
46 + engine_client: EngineClient,
47 model_config: ModelConfig,
48 served_model_names: List[str],
49 *,
@@ -52,7 +52,7 @@ class OpenAIServingCompletion(OpenAIServing):
52 request_logger: Optional[RequestLogger],
53 return_tokens_as_token_ids: bool = False,
54 ):
55 - super().__init__(async_engine_client=async_engine_client,
55 + super().__init__(engine_client=engine_client,
56 model_config=model_config,
57 served_model_names=served_model_names,
58 lora_modules=lora_modules,
@@ -78,6 +78,12 @@ class OpenAIServingCompletion(OpenAIServing):
78 if error_check_ret is not None:
79 return error_check_ret
80
81 + # If the engine is dead, raise the engine's DEAD_ERROR.
82 + # This is required for the streaming case, where we return a
83 + # success status before we actually start generating text :).
84 + if self.engine_client.errored:
85 + raise self.engine_client.dead_error
86 +
87 # Return error for unsupported features.
88 if request.suffix is not None:
89 return self.create_error_response(
@@ -95,8 +101,7 @@ class OpenAIServingCompletion(OpenAIServing):
101 prompt_adapter_request,
102 ) = self._maybe_get_adapters(request)
103
98 - tokenizer = await self.async_engine_client.get_tokenizer(
99 - lora_request)
104 + tokenizer = await self.engine_client.get_tokenizer(lora_request)
105
106 guided_decode_logits_processor = (
107 await self._guided_decode_logits_processor(request, tokenizer))
@@ -124,8 +129,8 @@ class OpenAIServingCompletion(OpenAIServing):
129 lora_request=lora_request,
130 prompt_adapter_request=prompt_adapter_request)
131
127 - is_tracing_enabled = (
128 - await self.async_engine_client.is_tracing_enabled())
132 + is_tracing_enabled = (await
133 + self.engine_client.is_tracing_enabled())
134 trace_headers = None
135 if is_tracing_enabled:
136 trace_headers = extract_trace_headers(raw_request.headers)
@@ -133,7 +138,7 @@ class OpenAIServingCompletion(OpenAIServing):
138 raw_request.headers):
139 log_tracing_disabled_warning()
140
136 - generator = self.async_engine_client.generate(
141 + generator = self.engine_client.generate(
142 {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
143 sampling_params,
144 request_id_item,
vllm/entrypoints/openai/serving_embedding.py
+5 -6
@@ -8,7 +8,7 @@ from fastapi import Request
8 from typing_extensions import assert_never
9
10 from vllm.config import ModelConfig
11 -from vllm.engine.protocol import AsyncEngineClient
11 +from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.logger import RequestLogger
13 from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
14 EmbeddingResponse,
@@ -71,13 +71,13 @@ class OpenAIServingEmbedding(OpenAIServing):
71
72 def __init__(
73 self,
74 - async_engine_client: AsyncEngineClient,
74 + engine_client: EngineClient,
75 model_config: ModelConfig,
76 served_model_names: List[str],
77 *,
78 request_logger: Optional[RequestLogger],
79 ):
80 - super().__init__(async_engine_client=async_engine_client,
80 + super().__init__(engine_client=engine_client,
81 model_config=model_config,
82 served_model_names=served_model_names,
83 lora_modules=None,
@@ -118,8 +118,7 @@ class OpenAIServingEmbedding(OpenAIServing):
118 prompt_adapter_request,
119 ) = self._maybe_get_adapters(request)
120
121 - tokenizer = await self.async_engine_client.get_tokenizer(
122 - lora_request)
121 + tokenizer = await self.engine_client.get_tokenizer(lora_request)
122
123 pooling_params = request.to_pooling_params()
124
@@ -144,7 +143,7 @@ class OpenAIServingEmbedding(OpenAIServing):
143 "Prompt adapter is not supported "
144 "for embedding models")
145
147 - generator = self.async_engine_client.encode(
146 + generator = self.engine_client.encode(
147 {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
148 pooling_params,
149 request_id_item,
vllm/entrypoints/openai/serving_engine.py
+4 -4
@@ -8,7 +8,7 @@ from pydantic import Field
8 from typing_extensions import Annotated
9
10 from vllm.config import ModelConfig
11 -from vllm.engine.protocol import AsyncEngineClient
11 +from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.logger import RequestLogger
13 # yapf conflicts with isort for this block
14 # yapf: disable
@@ -64,7 +64,7 @@ class OpenAIServing:
64
65 def __init__(
66 self,
67 - async_engine_client: AsyncEngineClient,
67 + engine_client: EngineClient,
68 model_config: ModelConfig,
69 served_model_names: List[str],
70 *,
@@ -75,7 +75,7 @@ class OpenAIServing:
75 ):
76 super().__init__()
77
78 - self.async_engine_client = async_engine_client
78 + self.engine_client = engine_client
79 self.model_config = model_config
80 self.max_model_len = model_config.max_model_len
81
@@ -159,7 +159,7 @@ class OpenAIServing:
159 async def _guided_decode_logits_processor(
160 self, request: Union[ChatCompletionRequest, CompletionRequest],
161 tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
162 - decoding_config = await self.async_engine_client.get_decoding_config()
162 + decoding_config = await self.engine_client.get_decoding_config()
163 guided_decoding_backend = request.guided_decoding_backend \
164 or decoding_config.guided_decoding_backend
165 return await get_guided_decoding_logits_processor(
vllm/entrypoints/openai/serving_tokenization.py
+5 -5
@@ -1,7 +1,7 @@
1 from typing import List, Optional, Union
2
3 from vllm.config import ModelConfig
4 -from vllm.engine.protocol import AsyncEngineClient
4 +from vllm.engine.protocol import EngineClient
5 from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
6 apply_mistral_chat_template,
7 load_chat_template,
@@ -29,7 +29,7 @@ class OpenAIServingTokenization(OpenAIServing):
29
30 def __init__(
31 self,
32 - async_engine_client: AsyncEngineClient,
32 + engine_client: EngineClient,
33 model_config: ModelConfig,
34 served_model_names: List[str],
35 *,
@@ -37,7 +37,7 @@ class OpenAIServingTokenization(OpenAIServing):
37 request_logger: Optional[RequestLogger],
38 chat_template: Optional[str],
39 ):
40 - super().__init__(async_engine_client=async_engine_client,
40 + super().__init__(engine_client=engine_client,
41 model_config=model_config,
42 served_model_names=served_model_names,
43 lora_modules=lora_modules,
@@ -66,7 +66,7 @@ class OpenAIServingTokenization(OpenAIServing):
66 prompt_adapter_request,
67 ) = self._maybe_get_adapters(request)
68
69 - tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
69 + tokenizer = await self.engine_client.get_tokenizer(lora_request)
70
71 prompt: Union[str, List[int]]
72 if isinstance(request, TokenizeChatRequest):
@@ -132,7 +132,7 @@ class OpenAIServingTokenization(OpenAIServing):
132 prompt_adapter_request,
133 ) = self._maybe_get_adapters(request)
134
135 - tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
135 + tokenizer = await self.engine_client.get_tokenizer(lora_request)
136
137 self._log_inputs(request_id,
138 request.tokens,
vllm/envs.py
+3 -3
@@ -57,7 +57,7 @@ if TYPE_CHECKING:
57 VERBOSE: bool = False
58 VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
59 VLLM_TEST_FORCE_FP8_MARLIN: bool = False
60 - VLLM_RPC_GET_DATA_TIMEOUT_MS: int = 5000
60 + VLLM_RPC_TIMEOUT: int = 10000 # ms
61 VLLM_PLUGINS: Optional[List[str]] = None
62 VLLM_TORCH_PROFILER_DIR: Optional[str] = None
63 VLLM_USE_TRITON_AWQ: bool = False
@@ -393,8 +393,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
393
394 # Time in ms for the zmq client to wait for a response from the backend
395 # server for simple data operations
396 - "VLLM_RPC_GET_DATA_TIMEOUT_MS":
397 - lambda: int(os.getenv("VLLM_RPC_GET_DATA_TIMEOUT_MS", "5000")),
396 + "VLLM_RPC_TIMEOUT":
397 + lambda: int(os.getenv("VLLM_RPC_TIMEOUT", "10000")),
398
399 # a list of plugin names to load, separated by commas.
400 # if this is not set, it means all plugins will be loaded
vllm/executor/cpu_executor.py
+1
@@ -106,6 +106,7 @@ class CPUExecutor(ExecutorBase):
106 )) for rank in range(1, world_size)
107 ]
108
109 + self.worker_monitor = None
110 if world_size != 1 or is_async:
111 if is_async:
112 async_worker_list = self.workers + [self.driver_worker]
vllm/executor/multiproc_worker_utils.py
+4
@@ -168,6 +168,8 @@ class ProcessWorkerWrapper:
168 self.tasks[task_id] = future
169 try:
170 self._task_queue.put((task_id, method, args, kwargs))
171 + except SystemExit:
172 + raise
173 except BaseException as e:
174 del self.tasks[task_id]
175 raise ChildProcessError("worker died") from e
@@ -222,6 +224,8 @@ def _run_worker_process(
224 try:
225 executor = getattr(worker, method)
226 output = executor(*args, **kwargs)
227 + except SystemExit:
228 + raise
229 except KeyboardInterrupt:
230 break
231 except BaseException as e: