| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Deploy DeepSeek R1 or V3 with Ray Serve LLM. |
| 4 | |
| 5 | Ray Serve LLM is a scalable and production-grade model serving library built |
| 6 | on the Ray distributed computing framework and first-class support for the vLLM engine. |
| 7 | |
| 8 | Key features: |
| 9 | - Automatic scaling, back-pressure, and load balancing across a Ray cluster. |
| 10 | - Unified multi-node multi-model deployment. |
| 11 | - Exposes an OpenAI-compatible HTTP API. |
| 12 | - Multi-LoRA support with shared base models. |
| 13 | |
| 14 | Run `python3 ray_serve_deepseek.py` to launch an endpoint. |
| 15 | |
| 16 | Learn more in the official Ray Serve LLM documentation: |
| 17 | https://docs.ray.io/en/latest/serve/llm/serving-llms.html |
| 18 | """ |
| 19 | |
| 20 | from ray import serve |
| 21 | from ray.serve.llm import LLMConfig, build_openai_app |
| 22 | |
| 23 | llm_config = LLMConfig( |
| 24 | model_loading_config={ |
| 25 | "model_id": "deepseek", |
| 26 | # Pre-downloading the model to local storage is recommended since |
| 27 | # the model is large. Set model_source="/path/to/the/model". |
| 28 | "model_source": "deepseek-ai/DeepSeek-R1", |
| 29 | }, |
| 30 | deployment_config={ |
| 31 | "autoscaling_config": { |
| 32 | "min_replicas": 1, |
| 33 | "max_replicas": 1, |
| 34 | } |
| 35 | }, |
| 36 | # Set to the node's accelerator type. |
| 37 | accelerator_type="H100", |
| 38 | # Customize engine arguments as required (for example, vLLM engine kwargs). |
| 39 | engine_kwargs={ |
| 40 | "tensor_parallel_size": 8, |
| 41 | "pipeline_parallel_size": 2, |
| 42 | "gpu_memory_utilization": 0.92, |
| 43 | "dtype": "auto", |
| 44 | "max_num_seqs": 40, |
| 45 | "max_model_len": 16384, |
| 46 | "enable_chunked_prefill": True, |
| 47 | "enable_prefix_caching": True, |
| 48 | "trust_remote_code": True, |
| 49 | }, |
| 50 | ) |
| 51 | |
| 52 | # Deploy the application. |
| 53 | llm_app = build_openai_app({"llm_configs": [llm_config]}) |
| 54 | serve.run(llm_app) |