1d694e78c9 · tduncan/Nvidia-Optimization-Stack-VLLM

.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh

-2

@@ -136,8 +136,6 @@ run_and_track_test 3 "test_accuracy.py::test_lm_eval_accuracy_v1_engine" \

136 "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_accuracy.py::test_lm_eval_accuracy_v1_engine"

137 run_and_track_test 4 "test_quantization_accuracy.py" \

138 "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py"

139 -run_and_track_test 5 "examples/offline_inference/tpu.py" \

140 - "python3 /workspace/vllm/examples/offline_inference/tpu.py"

139 run_and_track_test 6 "test_tpu_model_runner.py" \

140 "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py"

141 run_and_track_test 7 "test_sampler.py" \

.buildkite/test-amd.yaml

+6 -6

@@ -394,8 +394,8 @@ steps:

394 - python3 pooling/embed/vision_embedding_offline.py --seed 0

395 # Features demo

396 - python3 features/automatic_prefix_caching/prefix_caching_offline.py

397 - - python3 offline_inference/llm_engine_example.py

398 - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

397 + - python3 deployment/llm_engine_example.py

398 + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

399 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

400 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

401

@@ -1649,8 +1649,8 @@ steps:

1649 - python3 pooling/embed/vision_embedding_offline.py --seed 0

1650 # Features demo

1651 - python3 features/automatic_prefix_caching/prefix_caching_offline.py

1652 - - python3 offline_inference/llm_engine_example.py

1653 - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

1652 + - python3 deployment/llm_engine_example.py

1653 + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

1654 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

1655 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

1656

@@ -2930,8 +2930,8 @@ steps:

2930 - python3 pooling/embed/vision_embedding_offline.py --seed 0

2931 # Features demo

2932 - python3 features/automatic_prefix_caching/prefix_caching_offline.py

2933 - - python3 offline_inference/llm_engine_example.py

2934 - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

2933 + - python3 deployment/llm_engine_example.py

2934 + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

2935 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

2936 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

2937

.buildkite/test_areas/misc.yaml

+2 -2

@@ -117,8 +117,8 @@ steps:

117 - python3 pooling/embed/vision_embedding_offline.py --seed 0

118 # for features demo

119 - python3 features/automatic_prefix_caching/prefix_caching_offline.py

120 - - python3 offline_inference/llm_engine_example.py

121 - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

120 + - python3 deployment/llm_engine_example.py

121 + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

122 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

123 # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU

124 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

.buildkite/test_areas/model_runner_v2.yaml

+3 -3

@@ -37,7 +37,7 @@ steps:

37 - examples/generate/multimodal/

38 - examples/features/

39 - examples/pooling/embed/vision_embedding_offline.py

40 - - examples/others/tensorize_vllm_model.py

40 + - examples/features/tensorize_vllm_model.py

41 commands:

42 - set -x

43 - export VLLM_USE_V2_MODEL_RUNNER=1

@@ -55,8 +55,8 @@ steps:

55 - python3 pooling/embed/vision_embedding_offline.py --seed 0

56 # for features demo

57 - python3 features/automatic_prefix_caching/prefix_caching_offline.py

58 - - python3 offline_inference/llm_engine_example.py

59 - - python3 others/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 others/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

58 + - python3 deployment/llm_engine_example.py

59 + - python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 examples/features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors

60 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

61 # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU

62 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

docker/Dockerfile

+1 -1

@@ -860,7 +860,7 @@ LABEL org.opencontainers.image.source="https://github.com/vllm-project/vllm" \

860 # define sagemaker first, so it is not default from `docker build`

861 FROM vllm-openai-base AS vllm-sagemaker

862

863 -COPY examples/online_serving/sagemaker-entrypoint.sh .

863 +COPY examples/deployment/sagemaker-entrypoint.sh .

864 RUN chmod +x sagemaker-entrypoint.sh

865 ENTRYPOINT ["./sagemaker-entrypoint.sh"]

866

docs/contributing/model/transcription.md

+1 -1

@@ -278,7 +278,7 @@ Once your model implements `SupportsTranscription`, you can test the endpoints (

278 http://localhost:8000/v1/audio/translations

279 ```

280

281 -Or check out more examples in [examples/online_serving](../../../examples/online_serving).

281 +Or check out more examples in [examples/speech_to_text](../../../examples/speech_to_text).

282

283 !!! note

284 - If your model handles chunking internally (e.g., via its processor or encoder), set `min_energy_split_window_size=None` in the returned `SpeechToTextConfig` to disable server-side chunking.

docs/deployment/frameworks/helm.md

+1 -1

@@ -17,7 +17,7 @@ Before you begin, ensure that you have the following:

17

18 ## Installing the chart

19

20 -This guide uses the Helm chart at [examples/online_serving/chart-helm](../../../examples/online_serving/chart-helm).

20 +This guide uses the Helm chart at [examples/deployment/chart-helm](../../../examples/deployment/chart-helm).

21

22 To install the chart with the release name `test-vllm`:

23

docs/deployment/frameworks/lws.md

+2 -2

@@ -40,7 +40,7 @@ Deploy the following yaml file `lws.yaml`

40 command:

41 - sh

42 - -c

43 - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE);

43 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=$(LWS_GROUP_SIZE);

44 vllm serve meta-llama/Meta-Llama-3.1-405B-Instruct --port 8080 --tensor-parallel-size 8 --pipeline_parallel_size 2"

45 resources:

46 limits:

@@ -73,7 +73,7 @@ Deploy the following yaml file `lws.yaml`

73 command:

74 - sh

75 - -c

76 - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)"

76 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(LWS_LEADER_ADDRESS)"

77 resources:

78 limits:

79 nvidia.com/gpu: "8"

docs/deployment/frameworks/retrieval_augmented_generation.md

+2 -2

@@ -36,7 +36,7 @@ pip install -U vllm \

36 vllm serve qwen/Qwen1.5-0.5B-Chat --port 8001

37 ```

38

39 -1. Use the script: [examples/online_serving/retrieval_augmented_generation_with_langchain.py](../../../examples/online_serving/retrieval_augmented_generation_with_langchain.py)

39 +1. Use the script: [examples/applications/rag/retrieval_augmented_generation_with_langchain.py](../../../examples/applications/rag/retrieval_augmented_generation_with_langchain.py)

40

41 1. Run the script

42

@@ -74,7 +74,7 @@ pip install vllm \

74 vllm serve qwen/Qwen1.5-0.5B-Chat --port 8001

75 ```

76

77 -1. Use the script: [examples/online_serving/retrieval_augmented_generation_with_llamaindex.py](../../../examples/online_serving/retrieval_augmented_generation_with_llamaindex.py)

77 +1. Use the script: [examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py](../../../examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py)

78

79 1. Run the script:

80

docs/deployment/frameworks/skypilot.md

+2 -2

@@ -59,7 +59,7 @@ See the vLLM SkyPilot YAML for serving, [serving.yaml](https://github.com/skypil

59

60 echo 'Starting gradio server...'

61 git clone https://github.com/vllm-project/vllm.git || true

62 - python vllm/examples/online_serving/gradio_openai_chatbot_webserver.py \

62 + python vllm/examples/applications/chatbot/gradio_openai_chatbot_webserver.py \

63 -m $MODEL_NAME \

64 --port 8811 \

65 --model-url http://localhost:8081/v1 \

@@ -305,7 +305,7 @@ It is also possible to access the Llama-3 service with a separate GUI frontend,

305

306 echo 'Starting gradio server...'

307 git clone https://github.com/vllm-project/vllm.git || true

308 - python vllm/examples/online_serving/gradio_openai_chatbot_webserver.py \

308 + python vllm/examples/applications/api_client/gradio_openai_chatbot_webserver.py \

309 -m $MODEL_NAME \

310 --port 8811 \

311 --model-url http://$ENDPOINT/v1 \

docs/deployment/frameworks/streamlit.md

+1 -1

@@ -20,7 +20,7 @@ pip install vllm streamlit openai

20 vllm serve Qwen/Qwen1.5-0.5B-Chat

21 ```

22

23 -1. Use the script: [examples/online_serving/streamlit_openai_chatbot_webserver.py](../../../examples/online_serving/streamlit_openai_chatbot_webserver.py)

23 +1. Use the script: [examples/applications/chatbot/streamlit_openai_chatbot_webserver.py](../../../examples/applications/chatbot/streamlit_openai_chatbot_webserver.py)

24

25 1. Start the streamlit web UI and start to chat:

26

docs/deployment/integrations/kthena.md

+4 -4

@@ -78,7 +78,7 @@ Key points from the example YAML:

78 - sh

79 - -c

80 - >

81 - bash /vllm-workspace/examples/online_serving/multi-node-serving.sh leader --ray_cluster_size=2;

81 + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2;

82 python3 -m vllm.entrypoints.openai.api_server

83 --port 8080

84 --model meta-llama/Llama-3.1-405B-Instruct

@@ -93,7 +93,7 @@ Key points from the example YAML:

93 - sh

94 - -c

95 - >

96 - bash /vllm-workspace/examples/online_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)

96 + bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)

97 ```

98

99 ---

@@ -144,7 +144,7 @@ spec:

144 command:

145 - sh

146 - -c

147 - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh leader --ray_cluster_size=2;

147 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh leader --ray_cluster_size=2;

148 python3 -m vllm.entrypoints.openai.api_server --port 8080 --model meta-llama/Llama-3.1-405B-Instruct --tensor-parallel-size 8 --pipeline-parallel-size 2"

149 resources:

150 limits:

@@ -178,7 +178,7 @@ spec:

178 command:

179 - sh

180 - -c

181 - - "bash /vllm-workspace/examples/online_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)"

181 + - "bash /vllm-workspace/examples/ray_serving/multi-node-serving.sh worker --ray_address=$(ENTRY_ADDRESS)"

182 resources:

183 limits:

184 nvidia.com/gpu: "8"

docs/examples/README.md

+3 -3

@@ -2,6 +2,6 @@

2

3 vLLM's examples are split into three categories:

4

5 -- If you are using vLLM from within Python code, see the [Offline Inference](../../examples/offline_inference) section.

6 -- If you are using vLLM from an HTTP application or client, see the [Online Serving](../../examples/online_serving) section.

7 -- For examples of using some of vLLM's advanced features (e.g. LMCache or Tensorizer) which are not specific to either of the above use cases, see the [Others](../../examples/others) section.

5 +- If you are using vLLM from within Python code, see the [Offline Inference](.) section.

6 +- If you are using vLLM from an HTTP application or client, see the [Online Serving](.) section.

7 +- For examples of using some of vLLM's advanced features (e.g. LMCache or Tensorizer) which are not specific to either of the above use cases, see the [Others](.) section.

docs/features/quantization/auto_awq.md

+1 -1

@@ -47,7 +47,7 @@ After installing AutoAWQ, you are ready to quantize a model. Please refer to the

47 To run an AWQ model with vLLM, you can use [TheBloke/Llama-2-7b-Chat-AWQ](https://huggingface.co/TheBloke/Llama-2-7b-Chat-AWQ) with the following command:

48

49 ```bash

50 -python examples/offline_inference/llm_engine_example.py \

50 +python examples/deployment/llm_engine_example.py \

51 --model TheBloke/Llama-2-7b-Chat-AWQ \

52 --quantization awq

53 ```

docs/features/quantization/gptqmodel.md

+1 -1

@@ -58,7 +58,7 @@ Here is an example of how to quantize `meta-llama/Llama-3.2-1B-Instruct`:

58 To run an GPTQModel quantized model with vLLM, you can use [DeepSeek-R1-Distill-Qwen-7B-gptqmodel-4bit-vortex-v2](https://huggingface.co/ModelCloud/DeepSeek-R1-Distill-Qwen-7B-gptqmodel-4bit-vortex-v2) with the following command:

59

60 ```bash

61 -python examples/offline_inference/llm_engine_example.py \

61 +python examples/deployment/llm_engine_example.py \

62 --model ModelCloud/DeepSeek-R1-Distill-Qwen-7B-gptqmodel-4bit-vortex-v2

63 ```

64

docs/features/reasoning_outputs.md

+1 -1

@@ -157,7 +157,7 @@ OpenAI Python client library does not officially support `reasoning` attribute f

157 print(content, end="", flush=True)

158 ```

159

160 -Remember to check whether the `reasoning` exists in the response before accessing it. You could check out the [example](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/openai_chat_completion_with_reasoning_streaming.py).

160 +Remember to check whether the `reasoning` exists in the response before accessing it. You could check out the [example](https://github.com/vllm-project/vllm/blob/main/examples/reasoning/openai_chat_completion_with_reasoning_streaming.py).

161

162 ## Tool Calling

163

docs/getting_started/installation/gpu.xpu.inc.md

+1 -1

@@ -88,7 +88,7 @@ vllm serve facebook/opt-13b \

88 -tp=8

89 ```

90

91 -By default, a ray instance will be launched automatically if no existing one is detected in the system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring to the [examples/online_serving/run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/run_cluster.sh) helper script.

91 +By default, a ray instance will be launched automatically if no existing one is detected in the system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring to the [examples/ray_serving/run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/ray_serving/run_cluster.sh) helper script.

92

93 --8<-- [end:supported-features]

94 --8<-- [start:distributed-backend]

docs/models/extensions/tensorizer.md

+4 -4

@@ -14,7 +14,7 @@ To install `tensorizer`, run `pip install vllm[tensorizer]`.

14 ## The basics

15

16 To load a model using Tensorizer, the model first needs to be serialized by

17 -Tensorizer. [The example script](../../examples/others/tensorize_vllm_model.md) takes care of this process.

17 +Tensorizer. [The example script](../../../examples/features/tensorize_vllm_model.py) takes care of this process.

18

19 Let's walk through a basic example by serializing `facebook/opt-125m` using the script, and then loading it for inference.

20

@@ -25,7 +25,7 @@ CLI arguments. The docstring for the script itself explains the CLI args

25 and how to use it properly in great detail, and we'll use one of the examples from the docstring directly, assuming we want to serialize and save our model at our S3 bucket example `s3://my-bucket`:

26

27 ```bash

28 -python examples/others/tensorize_vllm_model.py \

28 +python examples/features/tensorize_vllm_model.py \

29 --model facebook/opt-125m \

30 serialize \

31 --serialized-directory s3://my-bucket \

@@ -35,7 +35,7 @@ python examples/others/tensorize_vllm_model.py \

35 This saves the model tensors at `s3://my-bucket/vllm/facebook/opt-125m/v1`. If you intend on applying a LoRA adapter to your tensorized model, you can pass the HF id of the LoRA adapter in the above command, and the artifacts will be saved there too:

36

37 ```bash

38 -python examples/others/tensorize_vllm_model.py \

38 +python examples/features/tensorize_vllm_model.py \

39 --model facebook/opt-125m \

40 --lora-path <lora_id> \

41 serialize \

@@ -71,7 +71,7 @@ llm = LLM(

71 As an example, CPU concurrency can be limited when serializing with `tensorizer` via the `limit_cpu_concurrency` parameter in the initializer for `TensorSerializer`. To set `limit_cpu_concurrency` to some arbitrary value, you would do so like this when serializing:

72

73 ```bash

74 -python examples/others/tensorize_vllm_model.py \

74 +python examples/features/tensorize_vllm_model.py \

75 --model facebook/opt-125m \

76 --lora-path <lora_id> \

77 serialize \

docs/serving/openai_compatible_server.md

+3 -3

@@ -251,7 +251,7 @@ The following extra parameters are supported:

251 Our Responses API is compatible with [OpenAI's Responses API](https://platform.openai.com/docs/api-reference/responses);

252 you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.

253

254 -Code example: [examples/online_serving/openai_responses_client_with_tools.py](../../examples/tool_calling/openai_responses_client_with_tools.py)

254 +Code example: [examples/tool_calling/openai_responses_client_with_tools.py](../../examples/tool_calling/openai_responses_client_with_tools.py)

255

256 #### Extra parameters

257

@@ -456,8 +456,8 @@ Audio must be sent as base64-encoded PCM16 audio at 16kHz sample rate, mono chan

456

457 #### Example Clients

458

459 -- [openai_realtime_client.py](https://github.com/vllm-project/vllm/tree/main/examples/online_serving/openai_realtime_client.py) - Upload and transcribe an audio file

460 -- [openai_realtime_microphone_client.py](https://github.com/vllm-project/vllm/tree/main/examples/online_serving/openai_realtime_microphone_client.py) - Gradio demo for live microphone transcription

459 +- [openai_realtime_client.py](https://github.com/vllm-project/vllm/tree/main/examples/speech_to_text/realtime/openai_realtime_client.py) - Upload and transcribe an audio file

460 +- [openai_realtime_microphone_client.py](https://github.com/vllm-project/vllm/tree/main/examples/speech_to_text/realtime/openai_realtime_microphone_client.py) - Gradio demo for live microphone transcription

461

462 ### Tokenizer API

463

examples/applications/chatbot/api_client.py renamed

examples/applications/chatbot/gradio_openai_chatbot_webserver.py renamed

+1 -1

@@ -5,7 +5,7 @@ Start vLLM API server:

5 vllm serve meta-llama/Llama-2-7b-chat-hf

6

7 Start Gradio OpenAI Chatbot Webserver:

8 - python examples/online_serving/gradio_openai_chatbot_webserver.py \

8 + python examples/applications/chatbot/gradio_openai_chatbot_webserver.py \

9 -m meta-llama/Llama-2-7b-chat-hf

10

11 Note that `pip install --upgrade gradio` is needed to run this example.

examples/applications/chatbot/gradio_webserver.py renamed

+1 -1

@@ -6,7 +6,7 @@ Start vLLM API server:

6 --model meta-llama/Llama-2-7b-chat-hf

7

8 Start Webserver:

9 - python examples/online_serving/gradio_webserver.py

9 + python examples/applications/chatbot/gradio_webserver.py

10

11 Note that `pip install --upgrade gradio` is needed to run this example.

12 More details: https://github.com/gradio-app/gradio

examples/applications/chatbot/streamlit_openai_chatbot_webserver.py renamed

examples/applications/rag/retrieval_augmented_generation_with_langchain.py renamed

examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py renamed

examples/deployment/async_llm_streaming.py renamed

+1 -1

@@ -8,7 +8,7 @@ token-by-token output in offline inference scenarios. It demonstrates DELTA mode

8 streaming where you receive new tokens as they are generated.

9

10 Usage:

11 - python examples/offline_inference/async_llm_streaming.py

11 + python examples/deployment/async_llm_streaming.py

12 """

13

14 import asyncio

examples/deployment/chart-helm/.helmignore renamed

examples/deployment/chart-helm/Chart.yaml renamed

examples/deployment/chart-helm/README.md renamed

examples/deployment/chart-helm/ct.yaml renamed

examples/deployment/chart-helm/lintconf.yaml renamed

examples/deployment/chart-helm/templates/_helpers.tpl renamed

examples/deployment/chart-helm/templates/configmap.yaml renamed

examples/deployment/chart-helm/templates/custom-objects.yaml renamed

examples/deployment/chart-helm/templates/deployment.yaml renamed

examples/deployment/chart-helm/templates/hpa.yaml renamed

examples/deployment/chart-helm/templates/job.yaml renamed

examples/deployment/chart-helm/templates/poddisruptionbudget.yaml renamed

examples/deployment/chart-helm/templates/pvc.yaml renamed

examples/deployment/chart-helm/templates/secrets.yaml renamed

examples/deployment/chart-helm/templates/service.yaml renamed

examples/deployment/chart-helm/tests/deployment_test.yaml renamed

examples/deployment/chart-helm/tests/job_test.yaml renamed

examples/deployment/chart-helm/tests/pvc_test.yaml renamed

examples/deployment/chart-helm/values.schema.json renamed

examples/deployment/chart-helm/values.yaml renamed

examples/deployment/llm_engine_example.py renamed

examples/deployment/sagemaker-entrypoint.sh renamed

examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py renamed

examples/features/logging_configuration.md renamed

examples/features/tensorize_vllm_model.py renamed

+5 -5

@@ -33,7 +33,7 @@ https://github.com/coreweave/tensorizer

33 To serialize a model, install vLLM from source, then run something

34 like this from the root level of this repository:

35

36 -python examples/others/tensorize_vllm_model.py \

36 +python examples/features/tensorize_vllm_model.py \

37 --model facebook/opt-125m \

38 serialize \

39 --serialized-directory s3://my-bucket \

@@ -53,7 +53,7 @@ providing a `--keyfile` argument.

53 To deserialize a model, you can run something like this from the root

54 level of this repository:

55

56 -python examples/others/tensorize_vllm_model.py \

56 +python examples/features/tensorize_vllm_model.py \

57 --model EleutherAI/gpt-j-6B \

58 --dtype float16 \

59 deserialize \

@@ -71,11 +71,11 @@ shard's rank. Sharded models serialized with this script will be named as

71 model-rank-%03d.tensors

72

73 For more information on the available arguments for serializing, run

74 -`python -m examples.others.tensorize_vllm_model serialize --help`.

74 +`python -m examples.features.tensorize_vllm_model serialize --help`.

75

76 Or for deserializing:

77

78 -`python examples/others/tensorize_vllm_model.py deserialize --help`.

78 +`python examples/features/tensorize_vllm_model.py deserialize --help`.

79

80 Once a model is serialized, tensorizer can be invoked with the `LLM` class

81 directly to load models:

@@ -100,7 +100,7 @@ vllm serve s3://my-bucket/vllm/facebook/opt-125m/v1 \

100 In order to see all of the available arguments usable to configure

101 loading with tensorizer that are given to `TensorizerConfig`, run:

102

103 -`python examples/others/tensorize_vllm_model.py deserialize --help`

103 +`python examples/features/tensorize_vllm_model.py deserialize --help`

104

105 under the `tensorizer options` section. These can also be used for

106 deserialization in this example script, although `--tensorizer-uri` and

examples/observability/dashboards/README.md

+1 -1

@@ -43,7 +43,7 @@ Both platforms provide equivalent monitoring capabilities:

43 First, navigate to this example's directory:

44

45 ```bash

46 -cd examples/online_serving/dashboards

46 +cd examples/observability/dashboards

47 ```

48

49 ### Grafana

examples/online_serving/utils.py deleted

-26

@@ -1,26 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -from openai import APIConnectionError, OpenAI

4 -from openai.pagination import SyncPage

5 -from openai.types.model import Model

6 -

7 -

8 -def get_first_model(client: OpenAI) -> str:

9 - """

10 - Get the first model from the vLLM server.

11 - """

12 - try:

13 - models: SyncPage[Model] = client.models.list()

14 - except APIConnectionError as e:

15 - raise RuntimeError(

16 - "Failed to get the list of models from the vLLM server at "

17 - f"{client.base_url} with API key {client.api_key}. Check\n"

18 - "1. the server is running\n"

19 - "2. the server URL is correct\n"

20 - "3. the API key is correct"

21 - ) from e

22 -

23 - if len(models.data) == 0:

24 - raise RuntimeError(f"No models found on the vLLM server at {client.base_url}")

25 -

26 - return models.data[0].id

examples/pooling/embed/openai_embedding_long_text/service.sh

+1 -1

@@ -119,7 +119,7 @@ echo " - API Key: $API_KEY"

119 echo " - Native Pooling: $POOLING_TYPE | Cross-chunk: MEAN"

120 echo ""

121 echo "🧪 Test the server with:"

122 -echo " python examples/online_serving/openai_embedding_long_text/client.py"

122 +echo " python examples/pooling/embed/openai_embedding_long_text/client.py"

123 echo ""

124 echo "📚 Enhanced features enabled:"

125 echo " ✅ Intelligent native pooling type detection"

examples/speech_to_text/lid/openai_lid_client.py

+3 -3

@@ -15,14 +15,14 @@ Start the vLLM server:

15 Then run this script:

16

17 # Use the built-in sample audio

18 - python examples/online_serving/openai_lid_client.py

18 + python examples/speech_to_text/lid/openai_lid_client.py

19

20 # Use your own audio file(s)

21 - python examples/online_serving/openai_lid_client.py \

21 + python examples/speech_to_text/lid/openai_lid_client.py \

22 --audio_paths audio_en.wav audio_zh.wav audio_fr.wav

23

24 # Batch-identify multiple files in one run

25 - python examples/online_serving/openai_lid_client.py \

25 + python examples/speech_to_text/lid/openai_lid_client.py \

26 --audio_paths /path/to/dir/*.wav

27

28 Requirements:

requirements/common.txt

+1 -1

@@ -41,7 +41,7 @@ compressed-tensors == 0.15.0.1 # required for compressed-tensors

41 depyf==0.20.0 # required for profiling and debugging with compilation config

42 cloudpickle # allows pickling lambda functions in model_executor/models/registry.py

43 watchfiles # required for http server to monitor the updates of TLS files

44 -python-json-logger # Used by logging as per examples/others/logging_configuration.md

44 +python-json-logger # Used by logging as per examples/features/logging_configuration.md

45 ninja # Required for xgrammar, rocm, tpu, xpu

46 pybase64 # fast base64 implementation

47 cbor2 # Required for cross-language serialization of hashable objects

tests/lora/test_llama_tp.py

+1 -1

@@ -184,7 +184,7 @@ def test_tp2_serialize_and_deserialize_lora(

184 result = subprocess.run(

185 [

186 sys.executable,

187 - f"{VLLM_PATH}/examples/others/tensorize_vllm_model.py",

187 + f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",

188 "--model",

189 MODEL_PATH,

190 "--lora-path",

tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py

+1 -1

@@ -460,7 +460,7 @@ async def test_serialize_and_serve_entrypoints(tmp_path):

460 result = subprocess.run(

461 [

462 sys.executable,

463 - f"{VLLM_PATH}/examples/others/tensorize_vllm_model.py",

463 + f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",

464 "--model",

465 model_ref,

466 "serialize",

vllm/assets/video.py

+1 -1

@@ -139,6 +139,6 @@ class VideoAsset:

139 """

140 Read audio data from the video asset, used in Qwen2.5-Omni examples.

141

142 - See also: examples/offline_inference/qwen2_5_omni/only_thinker.py

142 + See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py

143 """

144 return load_audio_pyav(self.video_path, sr=sampling_rate)[0]

vllm/model_executor/model_loader/tensorizer.py

+2 -2

@@ -211,7 +211,7 @@ class TensorizerConfig(MutableMapping):

211 encryption_keyfile: File path to a binary file containing a

212 binary key to use for decryption. `None` (the default) means

213 no decryption. See the example script in

214 - examples/others/tensorize_vllm_model.py.

214 + examples/features/tensorize_vllm_model.py.

215 s3_access_key_id: The access key for the S3 bucket. Can also be set via

216 the S3_ACCESS_KEY_ID environment variable.

217 s3_secret_access_key: The secret access key for the S3 bucket. Can also

@@ -579,7 +579,7 @@ def tensorizer_weights_iterator(

579 "loading on vLLM, as tensorizer is forced to load to CPU. "

580 "Consider deserializing a vLLM model instead for faster "

581 "load times. See the "

582 - "examples/others/tensorize_vllm_model.py example script "

582 + "examples/features/tensorize_vllm_model.py example script "

583 "for serializing vLLM models."

584 )

585

vllm/model_executor/model_loader/tensorizer_loader.py

+2 -2

@@ -73,7 +73,7 @@ class TensorizerLoader(BaseModelLoader):

73 """Load a serialized model with tensorizer to the CPU.

74

75 This is only necessary when the model isn't vLLM-tensorized (see

76 - examples/others/tensorize_vllm_model.py) This should still

76 + examples/features/tensorize_vllm_model.py) This should still

77 be faster than default HuggingFace loading, but will be slower than

78 loading a vLLM-tensorized model.

79 """

@@ -104,7 +104,7 @@ class TensorizerLoader(BaseModelLoader):

104 """Load serialized model weights with tensorizer.

105

106 Expects a vLLM-tensorized model. See the

107 - examples/others/tensorize_vllm_model.py example script

107 + examples/features/tensorize_vllm_model.py example script

108 for serializing vLLM models."""

109 if is_vllm_tensorized(self.tensorizer_config):

110 tensorizer_config = self._patch_tensorizer_config(model_config)

[Examples][last/6] Resettle examples. (#41084)