A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Docs] Improve documentation for Deepseek R1 on Ray Serve LLM (#20601)
Signed-off-by: Ricardo Decal <rdecal@anyscale.com>
Ricardo Decal committed
Jul 8, 2025 at 02:09 UTC
b91cb3fa5c40993a1e56ffb6915db9ffebf9aa0a
1 file changed
+19
-12
examples/online_serving/ray_serve_deepseek.py
+19
-12
@@ -1,13 +1,21 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""
4
-Example to deploy DeepSeek R1 or V3 with Ray Serve LLM.
5
-See more details at:
6
-https://docs.ray.io/en/latest/serve/tutorials/serve-deepseek.html
7
-And see Ray Serve LLM documentation at:
8
-https://docs.ray.io/en/latest/serve/llm/serving-llms.html
4
+Deploy DeepSeek R1 or V3 with Ray Serve LLM.
5
+
6
+Ray Serve LLM is a scalable and production-grade model serving library built
7
+on the Ray distributed computing framework and first-class support for the vLLM engine.
8
+
9
+Key features:
10
+- Automatic scaling, back-pressure, and load balancing across a Ray cluster.
11
+- Unified multi-node multi-model deployment.
12
+- Exposes an OpenAI-compatible HTTP API.
13
+- Multi-LoRA support with shared base models.
14
10
-Run `python3 ray_serve_deepseek.py` to deploy the model.
15
+Run `python3 ray_serve_deepseek.py` to launch an endpoint.
16
+
17
+Learn more in the official Ray Serve LLM documentation:
18
+https://docs.ray.io/en/latest/serve/llm/serving-llms.html
19
"""
20
21
from ray import serve
@@ -16,9 +24,8 @@ from ray.serve.llm import LLMConfig, build_openai_app
24
llm_config = LLMConfig(
25
model_loading_config={
26
"model_id": "deepseek",
19
- # Since DeepSeek model is huge, it is recommended to pre-download
20
- # the model to local disk, say /path/to/the/model and specify:
21
- # model_source="/path/to/the/model"
27
+ # Pre-downloading the model to local storage is recommended since
28
+ # the model is large. Set model_source="/path/to/the/model".
29
"model_source": "deepseek-ai/DeepSeek-R1",
30
},
31
deployment_config={
@@ -27,10 +34,10 @@ llm_config = LLMConfig(
34
"max_replicas": 1,
35
}
36
},
30
- # Change to the accelerator type of the node
37
+ # Set to the node's accelerator type.
38
accelerator_type="H100",
39
runtime_env={"env_vars": {"VLLM_USE_V1": "1"}},
33
- # Customize engine arguments as needed (e.g. vLLM engine kwargs)
40
+ # Customize engine arguments as required (for example, vLLM engine kwargs).
41
engine_kwargs={
42
"tensor_parallel_size": 8,
43
"pipeline_parallel_size": 2,
@@ -44,6 +51,6 @@ llm_config = LLMConfig(
51
},
52
)
53
47
-# Deploy the application
54
+# Deploy the application.
55
llm_app = build_openai_app({"llm_configs": [llm_config]})
56
serve.run(llm_app)