A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 18:45 UTC
[gRPC] Add standard gRPC health checking (grpc.health.v1) for Kubernetes native probes (#38016)
Signed-off-by: Honglin Cao <Caohonglin317@hotmail.com>
Honglin Cao committed
Apr 22, 2026 at 17:31 UTC
9c271f94039341101aa144fccc857230a86ac575
4 files changed
+199
-1
docs/deployment/k8s.md
+44
@@ -4,6 +4,7 @@ Deploying vLLM on Kubernetes is a scalable and efficient way to serve machine le
4
5
- [Deployment with CPUs](#deployment-with-cpus)
6
- [Deployment with GPUs](#deployment-with-gpus)
7
+- [Serving with gRPC](#serving-with-grpc)
8
- [Troubleshooting](#troubleshooting)
9
- [Startup Probe or Readiness Probe Failure, container log contains "KeyboardInterrupt: terminated"](#startup-probe-or-readiness-probe-failure-container-log-contains-keyboardinterrupt-terminated)
10
- [Conclusion](#conclusion)
@@ -387,6 +388,49 @@ INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
388
389
If the service is correctly deployed, you should receive a response from the vLLM model.
390
391
+## Serving with gRPC
392
+
393
+vLLM can serve models over gRPC instead of HTTP by passing the `--grpc` flag. This requires the optional gRPC dependencies:
394
+
395
+```bash
396
+pip install vllm[grpc]
397
+```
398
+
399
+When using `--grpc`, the server exposes the standard [gRPC Health Checking Protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) (`grpc.health.v1.Health`), which integrates with Kubernetes [native gRPC probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-grpc-liveness-probe) (available since Kubernetes 1.24).
400
+
401
+To deploy with gRPC, change the `vllm serve` command to include `--grpc` and replace `httpGet` probes with `grpc` probes:
402
+
403
+```yaml
404
+containers:
405
+- name: mistral-7b
406
+ image: vllm/vllm-openai:latest
407
+ command: ["/bin/sh", "-c"]
408
+ args: [
409
+ "pip install vllm[grpc] && vllm serve mistralai/Mistral-7B-Instruct-v0.3 --grpc --port 50051 --trust-remote-code"
410
+ ]
411
+ ports:
412
+ - containerPort: 50051
413
+ livenessProbe:
414
+ grpc:
415
+ port: 50051
416
+ initialDelaySeconds: 120
417
+ periodSeconds: 10
418
+ readinessProbe:
419
+ grpc:
420
+ port: 50051
421
+ initialDelaySeconds: 120
422
+ periodSeconds: 5
423
+```
424
+
425
+!!! note
426
+ The gRPC health service checks the engine status on every probe. If the engine is unhealthy or the server is shutting down, the probe returns `NOT_SERVING`.
427
+
428
+You can also verify the health service manually with `grpcurl`:
429
+
430
+```bash
431
+grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check
432
+```
433
+
434
## Troubleshooting
435
436
### Startup Probe or Readiness Probe Failure, container log contains "KeyboardInterrupt: terminated"
setup.py
+1
-1
@@ -1107,7 +1107,7 @@ setup(
1107
# - .buildkite/test-amd.yaml
1108
"helion": ["helion==1.0.0"],
1109
# Optional deps for gRPC server (vllm serve --grpc)
1110
- "grpc": ["smg-grpc-servicer[vllm] >= 0.5.0"],
1110
+ "grpc": ["smg-grpc-servicer[vllm] >= 0.5.2"],
1111
# Optional deps for OpenTelemetry tracing
1112
"otel": [
1113
"opentelemetry-sdk>=1.26.0",
tests/entrypoints/test_grpc_health.py
new
+143
@@ -0,0 +1,143 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from unittest.mock import AsyncMock, MagicMock, patch
4
+
5
+import pytest
6
+
7
+grpc = pytest.importorskip("grpc")
8
+health_pb2 = pytest.importorskip("grpc_health.v1.health_pb2")
9
+VllmHealthServicer = pytest.importorskip(
10
+ "smg_grpc_servicer.vllm.health_servicer"
11
+).VllmHealthServicer
12
+
13
+SERVING = health_pb2.HealthCheckResponse.SERVING
14
+NOT_SERVING = health_pb2.HealthCheckResponse.NOT_SERVING
15
+SERVICE_UNKNOWN = health_pb2.HealthCheckResponse.SERVICE_UNKNOWN
16
+
17
+
18
+@pytest.fixture
19
+def async_llm():
20
+ mock = MagicMock()
21
+ mock.check_health = AsyncMock()
22
+ return mock
23
+
24
+
25
+@pytest.fixture
26
+def context():
27
+ return MagicMock(spec=grpc.aio.ServicerContext)
28
+
29
+
30
+@pytest.fixture
31
+def servicer(async_llm):
32
+ return VllmHealthServicer(async_llm)
33
+
34
+
35
+@pytest.fixture
36
+def request_msg():
37
+ msg = MagicMock()
38
+ msg.service = ""
39
+ return msg
40
+
41
+
42
+# -- Check() tests --
43
+
44
+
45
+@pytest.mark.asyncio
46
+async def test_check_serving_overall(servicer, request_msg, context, async_llm):
47
+ request_msg.service = ""
48
+ response = await servicer.Check(request_msg, context)
49
+ assert response.status == SERVING
50
+ async_llm.check_health.assert_awaited_once()
51
+
52
+
53
+@pytest.mark.asyncio
54
+async def test_check_serving_vllm_service(servicer, request_msg, context, async_llm):
55
+ request_msg.service = "vllm.grpc.engine.VllmEngine"
56
+ response = await servicer.Check(request_msg, context)
57
+ assert response.status == SERVING
58
+ async_llm.check_health.assert_awaited_once()
59
+
60
+
61
+@pytest.mark.asyncio
62
+async def test_check_not_serving_engine_errored(
63
+ servicer, request_msg, context, async_llm
64
+):
65
+ async_llm.check_health = AsyncMock(side_effect=Exception("engine dead"))
66
+ request_msg.service = ""
67
+ response = await servicer.Check(request_msg, context)
68
+ assert response.status == NOT_SERVING
69
+
70
+
71
+@pytest.mark.asyncio
72
+async def test_check_not_serving_shutting_down(
73
+ servicer, request_msg, context, async_llm
74
+):
75
+ servicer.set_not_serving()
76
+ request_msg.service = ""
77
+ response = await servicer.Check(request_msg, context)
78
+ assert response.status == NOT_SERVING
79
+ async_llm.check_health.assert_not_awaited()
80
+
81
+
82
+@pytest.mark.asyncio
83
+async def test_check_unknown_service_status(servicer, request_msg, context):
84
+ request_msg.service = "nonexistent.Service"
85
+ response = await servicer.Check(request_msg, context)
86
+ assert response.status == SERVICE_UNKNOWN
87
+
88
+
89
+@pytest.mark.asyncio
90
+async def test_check_unknown_service_grpc_code(servicer, request_msg, context):
91
+ request_msg.service = "fake.Svc"
92
+ await servicer.Check(request_msg, context)
93
+ context.set_code.assert_called_once_with(grpc.StatusCode.NOT_FOUND)
94
+ context.set_details.assert_called_once()
95
+ details_arg = context.set_details.call_args[0][0]
96
+ assert "fake.Svc" in details_arg
97
+
98
+
99
+@pytest.mark.asyncio
100
+@patch("smg_grpc_servicer.vllm.health_servicer.logger")
101
+async def test_check_logs_exception_on_error(
102
+ mock_logger, servicer, request_msg, context, async_llm
103
+):
104
+ async_llm.check_health = AsyncMock(side_effect=Exception("engine exploded"))
105
+ request_msg.service = ""
106
+ await servicer.Check(request_msg, context)
107
+ mock_logger.exception.assert_called_once()
108
+ log_args = mock_logger.exception.call_args
109
+ assert "service" in str(log_args).lower()
110
+
111
+
112
+# -- Watch() tests --
113
+
114
+
115
+@pytest.mark.asyncio
116
+async def test_watch_yields_serving(servicer, request_msg, context, async_llm):
117
+ request_msg.service = ""
118
+ watch_iter = servicer.Watch(request_msg, context)
119
+ first = await anext(watch_iter.__aiter__())
120
+ assert first.status == SERVING
121
+
122
+
123
+@pytest.mark.asyncio
124
+async def test_watch_yields_not_serving(servicer, request_msg, context, async_llm):
125
+ async_llm.check_health = AsyncMock(side_effect=Exception("engine down"))
126
+ request_msg.service = ""
127
+ watch_iter = servicer.Watch(request_msg, context)
128
+ first = await anext(watch_iter.__aiter__())
129
+ assert first.status == NOT_SERVING
130
+
131
+
132
+@pytest.mark.asyncio
133
+async def test_watch_unknown_service(servicer, request_msg, context):
134
+ request_msg.service = "fake.Service"
135
+ results = []
136
+ async for response in servicer.Watch(request_msg, context):
137
+ results.append(response)
138
+ assert len(results) == 1
139
+ assert results[0].status == SERVICE_UNKNOWN
140
+ # Watch returns SERVICE_UNKNOWN in the response body (not as a gRPC error
141
+ # code) so the stream terminates normally -- unlike Check, which sets
142
+ # NOT_FOUND on the gRPC context for unknown services.
143
+ context.set_code.assert_not_called()
vllm/entrypoints/grpc_server.py
+11
@@ -26,8 +26,10 @@ import time
26
27
try:
28
import grpc
29
+ from grpc_health.v1 import health_pb2_grpc
30
from grpc_reflection.v1alpha import reflection
31
from smg_grpc_proto import vllm_engine_pb2, vllm_engine_pb2_grpc
32
+ from smg_grpc_servicer.vllm.health_servicer import VllmHealthServicer
33
from smg_grpc_servicer.vllm.servicer import VllmEngineServicer
34
except ImportError as e:
35
raise ImportError(
@@ -98,9 +100,14 @@ async def serve_grpc(args: argparse.Namespace):
100
# Add servicer to server
101
vllm_engine_pb2_grpc.add_VllmEngineServicer_to_server(servicer, server)
102
103
+ # Add standard gRPC health service for Kubernetes probes
104
+ health_servicer = VllmHealthServicer(async_llm)
105
+ health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
106
+
107
# Enable reflection for grpcurl and other tools
108
service_names = (
109
vllm_engine_pb2.DESCRIPTOR.services_by_name["VllmEngine"].full_name,
110
+ "grpc.health.v1.Health",
111
reflection.SERVICE_NAME,
112
)
113
reflection.enable_server_reflection(service_names, server)
@@ -147,6 +154,10 @@ async def serve_grpc(args: argparse.Namespace):
154
logger.info("Shutting down vLLM gRPC server...")
155
if stats_task is not None:
156
stats_task.cancel()
157
+ try:
158
+ health_servicer.set_not_serving()
159
+ except Exception: # broad: must not prevent server.stop() / shutdown()
160
+ logger.warning("Failed to set health status to NOT_SERVING", exc_info=True)
161
await server.stop(grace=5.0)
162
logger.info("gRPC server stopped")
163
async_llm.shutdown()