d00d652998 · tduncan/Nvidia-Optimization-Stack-VLLM

.buildkite/nightly-benchmarks/scripts/launch-server.sh

+2 -6

@@ -181,18 +181,14 @@ launch_vllm_server() {

181 if echo "$common_params" | jq -e 'has("fp8")' >/dev/null; then

182 echo "Key 'fp8' exists in common params. Use neuralmagic fp8 model for convenience."

183 model=$(echo "$common_params" | jq -r '.neuralmagic_quantized_model')

184 - server_command="python3 \

185 - -m vllm.entrypoints.openai.api_server \

184 + server_command="vllm serve $model \

185 -tp $tp \

187 - --model $model \

186 --port $port \

187 $server_args"

188 else

189 echo "Key 'fp8' does not exist in common params."

192 - server_command="python3 \

193 - -m vllm.entrypoints.openai.api_server \

190 + server_command="vllm serve $model \

191 -tp $tp \

195 - --model $model \

192 --port $port \

193 $server_args"

194 fi

.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh

+1 -2

@@ -365,8 +365,7 @@ run_serving_tests() {

365 continue

366 fi

367

368 - server_command="$server_envs python3 \

369 - -m vllm.entrypoints.openai.api_server \

368 + server_command="$server_envs vllm serve \

369 $server_args"

370

371 # run the server

.buildkite/scripts/run-benchmarks.sh

+1 -1

@@ -18,7 +18,7 @@ vllm bench throughput --input-len 256 --output-len 256 --output-json throughput_

18 bench_throughput_exit_code=$?

19

20 # run server-based benchmarks and upload the result to buildkite

21 -python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf &

21 +vllm serve meta-llama/Llama-2-7b-chat-hf &

22 server_pid=$!

23 wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json

24

benchmarks/disagg_benchmarks/disagg_overhead_benchmark.sh

+2 -6

@@ -55,9 +55,7 @@ benchmark() {

55 output_len=$2

56

57

58 - CUDA_VISIBLE_DEVICES=0 python3 \

59 - -m vllm.entrypoints.openai.api_server \

60 - --model $model \

58 + CUDA_VISIBLE_DEVICES=0 vllm serve $model \

59 --port 8100 \

60 --max-model-len 10000 \

61 --gpu-memory-utilization 0.6 \

@@ -65,9 +63,7 @@ benchmark() {

63 '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &

64

65

68 - CUDA_VISIBLE_DEVICES=1 python3 \

69 - -m vllm.entrypoints.openai.api_server \

70 - --model $model \

66 + CUDA_VISIBLE_DEVICES=1 vllm serve $model \

67 --port 8200 \

68 --max-model-len 10000 \

69 --gpu-memory-utilization 0.6 \

benchmarks/disagg_benchmarks/disagg_performance_benchmark.sh

+4 -12

@@ -38,16 +38,12 @@ wait_for_server() {

38 launch_chunked_prefill() {

39 model="meta-llama/Meta-Llama-3.1-8B-Instruct"

40 # disagg prefill

41 - CUDA_VISIBLE_DEVICES=0 python3 \

42 - -m vllm.entrypoints.openai.api_server \

43 - --model $model \

41 + CUDA_VISIBLE_DEVICES=0 vllm serve $model \

42 --port 8100 \

43 --max-model-len 10000 \

44 --enable-chunked-prefill \

45 --gpu-memory-utilization 0.6 &

48 - CUDA_VISIBLE_DEVICES=1 python3 \

49 - -m vllm.entrypoints.openai.api_server \

50 - --model $model \

46 + CUDA_VISIBLE_DEVICES=1 vllm serve $model \

47 --port 8200 \

48 --max-model-len 10000 \

49 --enable-chunked-prefill \

@@ -62,18 +58,14 @@ launch_chunked_prefill() {

58 launch_disagg_prefill() {

59 model="meta-llama/Meta-Llama-3.1-8B-Instruct"

60 # disagg prefill

65 - CUDA_VISIBLE_DEVICES=0 python3 \

66 - -m vllm.entrypoints.openai.api_server \

67 - --model $model \

61 + CUDA_VISIBLE_DEVICES=0 vllm serve $model \

62 --port 8100 \

63 --max-model-len 10000 \

64 --gpu-memory-utilization 0.6 \

65 --kv-transfer-config \

66 '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &

67

74 - CUDA_VISIBLE_DEVICES=1 python3 \

75 - -m vllm.entrypoints.openai.api_server \

76 - --model $model \

68 + CUDA_VISIBLE_DEVICES=1 vllm serve $model \

69 --port 8200 \

70 --max-model-len 10000 \

71 --gpu-memory-utilization 0.6 \

docker/Dockerfile

+1 -1

@@ -565,5 +565,5 @@ ENTRYPOINT ["./sagemaker-entrypoint.sh"]

565

566 FROM vllm-openai-base AS vllm-openai

567

568 -ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]

568 +ENTRYPOINT ["vllm", "serve"]

569 #################### OPENAI API SERVER ####################

docker/Dockerfile.cpu

+1 -1

@@ -177,4 +177,4 @@ RUN --mount=type=cache,target=/root/.cache/uv \

177 --mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \

178 uv pip install dist/*.whl

179

180 -ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]

180 +ENTRYPOINT ["vllm", "serve"]

docker/Dockerfile.ppc64le

+1 -1

@@ -314,4 +314,4 @@ WORKDIR /workspace/

314

315 RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks

316

317 -ENTRYPOINT ["python", "-m", "vllm.entrypoints.openai.api_server"]

317 +ENTRYPOINT ["vllm", "serve"]

docker/Dockerfile.s390x

+1 -1

@@ -309,4 +309,4 @@ USER 2000

309 WORKDIR /home/vllm

310

311 # Set the default entrypoint

312 -ENTRYPOINT ["python", "-m", "vllm.entrypoints.openai.api_server"]

312 +ENTRYPOINT ["vllm", "serve"]

docker/Dockerfile.xpu

+1 -1

@@ -69,4 +69,4 @@ RUN --mount=type=cache,target=/root/.cache/pip \

69

70 # install development dependencies (for testing)

71 RUN python3 -m pip install -e tests/vllm_test_utils

72 -ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]

72 +ENTRYPOINT ["vllm", "serve"]

docs/contributing/benchmarks.md

+2 -4

@@ -661,8 +661,7 @@ Benchmark the performance of multi-modal requests in vLLM.

661 Start vLLM:

662

663 ```bash

664 -python -m vllm.entrypoints.openai.api_server \

665 - --model Qwen/Qwen2.5-VL-7B-Instruct \

664 +vllm serve Qwen/Qwen2.5-VL-7B-Instruct \

665 --dtype bfloat16 \

666 --limit-mm-per-prompt '{"image": 1}' \

667 --allowed-local-media-path /path/to/sharegpt4v/images

@@ -688,8 +687,7 @@ vllm bench serve \

687 Start vLLM:

688

689 ```bash

691 -python -m vllm.entrypoints.openai.api_server \

692 - --model Qwen/Qwen2.5-VL-7B-Instruct \

690 +vllm serve Qwen/Qwen2.5-VL-7B-Instruct \

691 --dtype bfloat16 \

692 --limit-mm-per-prompt '{"video": 1}' \

693 --allowed-local-media-path /path/to/sharegpt4video/videos

docs/contributing/profiling.md

+1 -2

@@ -39,8 +39,7 @@ Refer to <gh-file:examples/offline_inference/simple_profiling.py> for an example

39

40 ```bash

41 VLLM_TORCH_PROFILER_DIR=./vllm_profile \

42 - python -m vllm.entrypoints.openai.api_server \

43 - --model meta-llama/Meta-Llama-3-70B

42 + vllm serve meta-llama/Meta-Llama-3-70B

43 ```

44

45 vllm bench command:

docs/deployment/frameworks/autogen.md

+1 -2

@@ -19,8 +19,7 @@ pip install -U "autogen-agentchat" "autogen-ext[openai]"

19 1. Start the vLLM server with the supported chat completion model, e.g.

20

21 ```bash

22 - python -m vllm.entrypoints.openai.api_server \

23 - --model mistralai/Mistral-7B-Instruct-v0.2

22 + vllm serve mistralai/Mistral-7B-Instruct-v0.2

23 ```

24

25 1. Call it with AutoGen:

docs/deployment/frameworks/open-webui.md

+1 -1

@@ -20,7 +20,7 @@ To get started with Open WebUI using vLLM, follow these steps:

20 For example:

21

22 ```console

23 - python -m vllm.entrypoints.openai.api_server --host 0.0.0.0 --port 8000

23 + vllm serve <model> --host 0.0.0.0 --port 8000

24 ```

25

26 3. Start the Open WebUI Docker container:

docs/deployment/frameworks/skypilot.md

+6 -6

@@ -32,6 +32,7 @@ See the vLLM SkyPilot YAML for serving, [serving.yaml](https://github.com/skypil

32 ports: 8081 # Expose to internet traffic.

33

34 envs:

35 + PYTHONUNBUFFERED: 1

36 MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct

37 HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.

38

@@ -47,9 +48,8 @@ See the vLLM SkyPilot YAML for serving, [serving.yaml](https://github.com/skypil

48 run: |

49 conda activate vllm

50 echo 'Starting vllm api server...'

50 - python -u -m vllm.entrypoints.openai.api_server \

51 + vllm serve $MODEL_NAME \

52 --port 8081 \

52 - --model $MODEL_NAME \

53 --trust-remote-code \

54 --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \

55 2>&1 | tee api_server.log &

@@ -131,6 +131,7 @@ SkyPilot can scale up the service to multiple service replicas with built-in aut

131 ports: 8081 # Expose to internet traffic.

132

133 envs:

134 + PYTHONUNBUFFERED: 1

135 MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct

136 HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.

137

@@ -146,9 +147,8 @@ SkyPilot can scale up the service to multiple service replicas with built-in aut

147 run: |

148 conda activate vllm

149 echo 'Starting vllm api server...'

149 - python -u -m vllm.entrypoints.openai.api_server \

150 + vllm serve $MODEL_NAME \

151 --port 8081 \

151 - --model $MODEL_NAME \

152 --trust-remote-code \

153 --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \

154 2>&1 | tee api_server.log

@@ -243,6 +243,7 @@ This will scale the service up to when the QPS exceeds 2 for each replica.

243 ports: 8081 # Expose to internet traffic.

244

245 envs:

246 + PYTHONUNBUFFERED: 1

247 MODEL_NAME: meta-llama/Meta-Llama-3-8B-Instruct

248 HF_TOKEN: <your-huggingface-token> # Change to your own huggingface token, or use --env to pass.

249

@@ -258,9 +259,8 @@ This will scale the service up to when the QPS exceeds 2 for each replica.

259 run: |

260 conda activate vllm

261 echo 'Starting vllm api server...'

261 - python -u -m vllm.entrypoints.openai.api_server \

262 + vllm serve $MODEL_NAME \

263 --port 8081 \

263 - --model $MODEL_NAME \

264 --trust-remote-code \

265 --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \

266 2>&1 | tee api_server.log

docs/design/arch_overview.md

+5

@@ -69,6 +69,11 @@ Sometimes you may see the API server entrypoint used directly instead of via the

69 python -m vllm.entrypoints.openai.api_server --model <model>

70 ```

71

72 +!!! warning

73 +

74 + `python -m vllm.entrypoints.openai.api_server` is deprecated

75 + and may become unsupported in a future release.

76 +

77 That code can be found in <gh-file:vllm/entrypoints/openai/api_server.py>.

78

79 More details on the API server can be found in the [OpenAI-Compatible Server](../serving/openai_compatible_server.md) document.

docs/features/sleep_mode.md

+1 -2

@@ -64,8 +64,7 @@ To enable sleep mode in a vLLM server you need to initialize it with the flag `V

64 When using the flag `VLLM_SERVER_DEV_MODE=1` you enable development endpoints, and these endpoints should not be exposed to users.

65

66 ```bash

67 -VLLM_SERVER_DEV_MODE=1 python -m vllm.entrypoints.openai.api_server \

68 - --model Qwen/Qwen3-0.6B \

67 +VLLM_SERVER_DEV_MODE=1 vllm serve Qwen/Qwen3-0.6B \

68 --enable-sleep-mode \

69 --port 8000

70 ```

docs/features/spec_decode.md

+1 -2

@@ -48,10 +48,9 @@ The following code configures vLLM in an offline mode to use speculative decodin

48 To perform the same with an online mode launch the server:

49

50 ```bash

51 -python -m vllm.entrypoints.openai.api_server \

51 +vllm serve facebook/opt-6.7b \

52 --host 0.0.0.0 \

53 --port 8000 \

54 - --model facebook/opt-6.7b \

54 --seed 42 \

55 -tp 1 \

56 --gpu_memory_utilization 0.8 \

docs/getting_started/installation/gpu/xpu.inc.md

+1 -2

@@ -67,8 +67,7 @@ docker run -it \

67 XPU platform supports **tensor parallel** inference/serving and also supports **pipeline parallel** as a beta feature for online serving. For **pipeline parallel**, we support it on single node with mp as the backend. For example, a reference execution like following:

68

69 ```bash

70 -python -m vllm.entrypoints.openai.api_server \

71 - --model=facebook/opt-13b \

70 +vllm serve facebook/opt-13b \

71 --dtype=bfloat16 \

72 --max_model_len=1024 \

73 --distributed-executor-backend=mp \

examples/online_serving/sagemaker-entrypoint.sh

+1 -1

@@ -21,4 +21,4 @@ while IFS='=' read -r key value; do

21 done < <(env | grep "^${PREFIX}")

22

23 # Pass the collected arguments to the main entrypoint

24 -exec python3 -m vllm.entrypoints.openai.api_server "${ARGS[@]}"

\ No newline at end of file

24 +exec vllm serve "${ARGS[@]}"

\ No newline at end of file

tests/utils_/test_utils.py

+37 -7

@@ -786,13 +786,43 @@ def test_model_specification(parser_with_config, cli_config_file,

786 parser_with_config.parse_args(['serve', '--config', cli_config_file])

787

788 # Test using --model option raises error

789 - with pytest.raises(

790 - ValueError,

791 - match=

792 - ("With `vllm serve`, you should provide the model as a positional "

793 - "argument or in a config file instead of via the `--model` option."),

794 - ):

795 - parser_with_config.parse_args(['serve', '--model', 'my-model'])

789 + # with pytest.raises(

790 + # ValueError,

791 + # match=

792 + # ("With `vllm serve`, you should provide the model as a positional "

793 + # "argument or in a config file instead of via the `--model` option."),

794 + # ):

795 + # parser_with_config.parse_args(['serve', '--model', 'my-model'])

796 +

797 + # Test using --model option back-compatibility

798 + # (when back-compatibility ends, the above test should be uncommented

799 + # and the below test should be removed)

800 + args = parser_with_config.parse_args([

801 + 'serve',

802 + '--tensor-parallel-size',

803 + '2',

804 + '--model',

805 + 'my-model',

806 + '--trust-remote-code',

807 + '--port',

808 + '8001',

809 + ])

810 + assert args.model is None

811 + assert args.tensor_parallel_size == 2

812 + assert args.trust_remote_code is True

813 + assert args.port == 8001

814 +

815 + args = parser_with_config.parse_args([

816 + 'serve',

817 + '--tensor-parallel-size=2',

818 + '--model=my-model',

819 + '--trust-remote-code',

820 + '--port=8001',

821 + ])

822 + assert args.model is None

823 + assert args.tensor_parallel_size == 2

824 + assert args.trust_remote_code is True

825 + assert args.port == 8001

826

827 # Test other config values are preserved

828 args = parser_with_config.parse_args([

vllm/utils/__init__.py

+29 -5

@@ -1855,13 +1855,37 @@ class FlexibleArgumentParser(ArgumentParser):

1855

1856 # Check for --model in command line arguments first

1857 if args and args[0] == "serve":

1858 - model_in_cli_args = any(arg == '--model' for arg in args)

1859 -

1860 - if model_in_cli_args:

1861 - raise ValueError(

1858 + try:

1859 + model_idx = next(

1860 + i for i, arg in enumerate(args)

1861 + if arg == "--model" or arg.startswith("--model="))

1862 + logger.warning(

1863 "With `vllm serve`, you should provide the model as a "

1864 "positional argument or in a config file instead of via "

1864 - "the `--model` option.")

1865 + "the `--model` option. "

1866 + "The `--model` option will be removed in v0.13.")

1867 +

1868 + if args[model_idx] == "--model":

1869 + model_tag = args[model_idx + 1]

1870 + rest_start_idx = model_idx + 2

1871 + else:

1872 + model_tag = args[model_idx].removeprefix("--model=")

1873 + rest_start_idx = model_idx + 1

1874 +

1875 + # Move <model> to the front, e,g:

1876 + # [Before]

1877 + # vllm serve -tp 2 --model <model> --enforce-eager --port 8001

1878 + # [After]

1879 + # vllm serve <model> -tp 2 --enforce-eager --port 8001

1880 + args = [

1881 + "serve",

1882 + model_tag,

1883 + *args[1:model_idx],

1884 + *args[rest_start_idx:],

1885 + ]

1886 + print("args", args)

1887 + except StopIteration:

1888 + pass

1889

1890 if '--config' in args:

1891 args = self._pull_args_from_config(args)

[CI/Build] Replace `vllm.entrypoints.openai.api_server` entrypoint with `vllm serve` command (#25967)