@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aba8d6ee00
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Doc] Move examples into categories (#11840)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Jan 8, 2025 at 13:09 UTC aba8d6ee006b78149ac4514f460e4038b2d4f607
116 files changed +153 -124
.buildkite/run-cpu-test.sh
+1 -1
@@ -30,7 +30,7 @@ function cpu_tests() {
30 # offline inference
31 docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" bash -c "
32 set -e
33 - python3 examples/offline_inference.py"
33 + python3 examples/offline_inference/offline_inference.py"
34
35 # Run basic model test
36 docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
.buildkite/run-gh200-test.sh
+1 -1
@@ -24,5 +24,5 @@ remove_docker_container
24
25 # Run the image and test offline inference
26 docker run --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
27 - python3 examples/offline_inference.py
27 + python3 examples/offline_inference/offline_inference.py
28 '
.buildkite/run-hpu-test.sh
+1 -1
@@ -13,4 +13,4 @@ trap remove_docker_container EXIT
13 remove_docker_container
14
15 # Run the image and launch offline inference
16 -docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference.py
\ No newline at end of file
16 +docker run --runtime=habana --name=hpu-test --network=host -e HABANA_VISIBLE_DEVICES=all -e VLLM_SKIP_WARMUP=true --entrypoint="" hpu-test-env python3 examples/offline_inference/offline_inference.py
\ No newline at end of file
.buildkite/run-neuron-test.sh
+1 -1
@@ -51,4 +51,4 @@ docker run --rm -it --device=/dev/neuron0 --device=/dev/neuron1 --network host \
51 -e "NEURON_COMPILE_CACHE_URL=${NEURON_COMPILE_CACHE_MOUNT}" \
52 --name "${container_name}" \
53 ${image_name} \
54 - /bin/bash -c "python3 /workspace/vllm/examples/offline_inference_neuron.py"
54 + /bin/bash -c "python3 /workspace/vllm/examples/offline_inference/offline_inference_neuron.py"
.buildkite/run-openvino-test.sh
+1 -1
@@ -13,4 +13,4 @@ trap remove_docker_container EXIT
13 remove_docker_container
14
15 # Run the image and launch offline inference
16 -docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference.py
16 +docker run --network host --env VLLM_OPENVINO_KVCACHE_SPACE=1 --name openvino-test openvino-test python3 /workspace/examples/offline_inference/offline_inference.py
.buildkite/run-tpu-test.sh
+1 -1
@@ -14,4 +14,4 @@ remove_docker_container
14 # For HF_TOKEN.
15 source /etc/environment
16 # Run a simple end-to-end example.
17 -docker run --privileged --net host --shm-size=16G -it -e "HF_TOKEN=$HF_TOKEN" --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
17 +docker run --privileged --net host --shm-size=16G -it -e "HF_TOKEN=$HF_TOKEN" --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 -m pip install pytest && python3 -m pip install lm_eval[api]==0.4.4 && pytest -v -s /workspace/vllm/tests/entrypoints/openai/test_accuracy.py && pytest -v -s /workspace/vllm/tests/tpu/test_custom_dispatcher.py && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference/offline_inference_tpu.py"
.buildkite/run-xpu-test.sh
+2 -2
@@ -14,6 +14,6 @@ remove_docker_container
14
15 # Run the image and test offline inference/tensor parallel
16 docker run --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path --entrypoint="" xpu-test sh -c '
17 - python3 examples/offline_inference.py
18 - python3 examples/offline_inference_cli.py -tp 2
17 + python3 examples/offline_inference/offline_inference.py
18 + python3 examples/offline_inference/offline_inference_cli.py -tp 2
19 '
.buildkite/test-pipeline.yaml
+13 -13
@@ -187,19 +187,19 @@ steps:
187 - examples/
188 commands:
189 - pip install tensorizer # for tensorizer test
190 - - python3 offline_inference.py
191 - - python3 cpu_offload.py
192 - - python3 offline_inference_chat.py
193 - - python3 offline_inference_with_prefix.py
194 - - python3 llm_engine_example.py
195 - - python3 offline_inference_vision_language.py
196 - - python3 offline_inference_vision_language_multi_image.py
197 - - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
198 - - python3 offline_inference_encoder_decoder.py
199 - - python3 offline_inference_classification.py
200 - - python3 offline_inference_embedding.py
201 - - python3 offline_inference_scoring.py
202 - - python3 offline_profile.py --model facebook/opt-125m run_num_steps --num-steps 2
190 + - python3 offline_inference/offline_inference.py
191 + - python3 offline_inference/cpu_offload.py
192 + - python3 offline_inference/offline_inference_chat.py
193 + - python3 offline_inference/offline_inference_with_prefix.py
194 + - python3 offline_inference/llm_engine_example.py
195 + - python3 offline_inference/offline_inference_vision_language.py
196 + - python3 offline_inference/offline_inference_vision_language_multi_image.py
197 + - python3 other/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 other/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
198 + - python3 offline_inference/offline_inference_encoder_decoder.py
199 + - python3 offline_inference/offline_inference_classification.py
200 + - python3 offline_inference/offline_inference_embedding.py
201 + - python3 offline_inference/offline_inference_scoring.py
202 + - python3 offline_inference/offline_profile.py --model facebook/opt-125m run_num_steps --num-steps 2
203
204 - label: Prefix Caching Test # 9min
205 mirror_hardwares: [amd]
.github/workflows/lint-and-deploy.yaml
+2 -2
@@ -27,7 +27,7 @@ jobs:
27 version: v3.10.1
28
29 - name: Run chart-testing (lint)
30 - run: ct lint --target-branch ${{ github.event.repository.default_branch }} --chart-dirs examples/chart-helm --charts examples/chart-helm
30 + run: ct lint --target-branch ${{ github.event.repository.default_branch }} --chart-dirs examples/online_serving/chart-helm --charts examples/online_serving/chart-helm
31
32 - name: Setup minio
33 run: |
@@ -64,7 +64,7 @@ jobs:
64 run: |
65 export AWS_ACCESS_KEY_ID=minioadmin
66 export AWS_SECRET_ACCESS_KEY=minioadmin
67 - helm install --wait --wait-for-jobs --timeout 5m0s --debug --create-namespace --namespace=ns-vllm test-vllm examples/chart-helm -f examples/chart-helm/values.yaml --set secrets.s3endpoint=http://minio:9000 --set secrets.s3bucketname=testbucket --set secrets.s3accesskeyid=$AWS_ACCESS_KEY_ID --set secrets.s3accesskey=$AWS_SECRET_ACCESS_KEY --set resources.requests.cpu=1 --set resources.requests.memory=4Gi --set resources.limits.cpu=2 --set resources.limits.memory=5Gi --set image.env[0].name=VLLM_CPU_KVCACHE_SPACE --set image.env[1].name=VLLM_LOGGING_LEVEL --set-string image.env[0].value="1" --set-string image.env[1].value="DEBUG" --set-string extraInit.s3modelpath="opt-125m/" --set-string 'resources.limits.nvidia\.com/gpu=0' --set-string 'resources.requests.nvidia\.com/gpu=0' --set-string image.repository="vllm-cpu-env"
67 + helm install --wait --wait-for-jobs --timeout 5m0s --debug --create-namespace --namespace=ns-vllm test-vllm examples/online_serving/chart-helm -f examples/online_serving/chart-helm/values.yaml --set secrets.s3endpoint=http://minio:9000 --set secrets.s3bucketname=testbucket --set secrets.s3accesskeyid=$AWS_ACCESS_KEY_ID --set secrets.s3accesskey=$AWS_SECRET_ACCESS_KEY --set resources.requests.cpu=1 --set resources.requests.memory=4Gi --set resources.limits.cpu=2 --set resources.limits.memory=5Gi --set image.env[0].name=VLLM_CPU_KVCACHE_SPACE --set image.env[1].name=VLLM_LOGGING_LEVEL --set-string image.env[0].value="1" --set-string image.env[1].value="DEBUG" --set-string extraInit.s3modelpath="opt-125m/" --set-string 'resources.limits.nvidia\.com/gpu=0' --set-string 'resources.requests.nvidia\.com/gpu=0' --set-string image.repository="vllm-cpu-env"
68
69 - name: curl test
70 run: |
Dockerfile
+1 -1
@@ -250,7 +250,7 @@ ENV VLLM_USAGE_SOURCE production-docker-image
250 # define sagemaker first, so it is not default from `docker build`
251 FROM vllm-openai-base AS vllm-sagemaker
252
253 -COPY examples/sagemaker-entrypoint.sh .
253 +COPY examples/online_serving/sagemaker-entrypoint.sh .
254 RUN chmod +x sagemaker-entrypoint.sh
255 ENTRYPOINT ["./sagemaker-entrypoint.sh"]
256
docs/source/contributing/profiling/profiling_index.md
+1 -1
@@ -26,7 +26,7 @@ Set the env variable VLLM_RPC_TIMEOUT to a big number before you start the serve
26
27 ### Offline Inference
28
29 -Refer to <gh-file:examples/offline_inference_with_profiler.py> for an example.
29 +Refer to <gh-file:examples/offline_inference/offline_inference_with_profiler.py> for an example.
30
31 ### OpenAI Server
32
docs/source/deployment/frameworks/skypilot.md
+2 -2
@@ -61,7 +61,7 @@ run: |
61
62 echo 'Starting gradio server...'
63 git clone https://github.com/vllm-project/vllm.git || true
64 - python vllm/examples/gradio_openai_chatbot_webserver.py \
64 + python vllm/examples/online_serving/gradio_openai_chatbot_webserver.py \
65 -m $MODEL_NAME \
66 --port 8811 \
67 --model-url http://localhost:8081/v1 \
@@ -321,7 +321,7 @@ run: |
321
322 echo 'Starting gradio server...'
323 git clone https://github.com/vllm-project/vllm.git || true
324 - python vllm/examples/gradio_openai_chatbot_webserver.py \
324 + python vllm/examples/online_serving/gradio_openai_chatbot_webserver.py \
325 -m $MODEL_NAME \
326 --port 8811 \
327 --model-url http://$ENDPOINT/v1 \
docs/source/features/disagg_prefill.md
+1 -1
@@ -21,7 +21,7 @@ Disaggregated prefill DOES NOT improve throughput.
21
22 ## Usage example
23
24 -Please refer to `examples/disaggregated_prefill.sh` for the example usage of disaggregated prefilling.
24 +Please refer to `examples/online_serving/disaggregated_prefill.sh` for the example usage of disaggregated prefilling.
25
26 ## Benchmarks
27
docs/source/features/lora.md
+1 -1
@@ -47,7 +47,7 @@ outputs = llm.generate(
47 )
48 ```
49
50 -Check out <gh-file:examples/multilora_inference.py> for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
50 +Check out <gh-file:examples/offline_inference/multilora_inference.py> for an example of how to use LoRA adapters with the async engine and how to use more advanced configuration options.
51
52 ## Serving LoRA Adapters
53
docs/source/features/quantization/auto_awq.md
+1 -1
@@ -47,7 +47,7 @@ print(f'Model is quantized and saved at "{quant_path}"')
47 To run an AWQ model with vLLM, you can use [TheBloke/Llama-2-7b-Chat-AWQ](https://huggingface.co/TheBloke/Llama-2-7b-Chat-AWQ) with the following command:
48
49 ```console
50 -$ python examples/llm_engine_example.py --model TheBloke/Llama-2-7b-Chat-AWQ --quantization awq
50 +$ python examples/offline_inference/llm_engine_example.py --model TheBloke/Llama-2-7b-Chat-AWQ --quantization awq
51 ```
52
53 AWQ models are also supported directly through the LLM entrypoint:
docs/source/features/quantization/fp8_e4m3_kvcache.md
+1 -1
@@ -28,7 +28,7 @@ Here is an example of how to enable this feature:
28
29 ```python
30 # two float8_e4m3fn kv cache scaling factor files are provided under tests/fp8_kv, please refer to
31 -# https://github.com/vllm-project/vllm/blob/main/examples/fp8/README.md to generate kv_cache_scales.json of your own.
31 +# https://github.com/vllm-project/vllm/blob/main/examples/other/fp8/README.md to generate kv_cache_scales.json of your own.
32
33 from vllm import LLM, SamplingParams
34 sampling_params = SamplingParams(temperature=1.3, top_p=0.8)
docs/source/features/structured_outputs.md
+2 -2
@@ -131,7 +131,7 @@ completion = client.chat.completions.create(
131 print(completion.choices[0].message.content)
132 ```
133
134 -Full example: <gh-file:examples/openai_chat_completion_structured_outputs.py>
134 +Full example: <gh-file:examples/online_serving/openai_chat_completion_structured_outputs.py>
135
136 ## Experimental Automatic Parsing (OpenAI API)
137
@@ -257,4 +257,4 @@ outputs = llm.generate(
257 print(outputs[0].outputs[0].text)
258 ```
259
260 -Full example: <gh-file:examples/offline_inference_structured_outputs.py>
260 +Full example: <gh-file:examples/offline_inference/offline_inference_structured_outputs.py>
docs/source/generate_examples.py
+25 -20
@@ -12,6 +12,7 @@ EXAMPLE_DOC_DIR = ROOT_DIR / "docs/source/getting_started/examples"
12 def fix_case(text: str) -> str:
13 subs = {
14 "api": "API",
15 + "Cli": "CLI",
16 "cpu": "CPU",
17 "llm": "LLM",
18 "tpu": "TPU",
@@ -58,7 +59,7 @@ class Index:
59 content = f"# {self.title}\n\n{self.description}\n\n"
60 content += "```{toctree}\n"
61 content += f":caption: {self.caption}\n:maxdepth: {self.maxdepth}\n"
61 - content += "\n".join(sorted(self.documents)) + "\n```\n"
62 + content += "\n".join(self.documents) + "\n```\n"
63 return content
64
65
@@ -131,11 +132,14 @@ class Example:
132 ROOT_DIR)
133
134 content = f"Source <gh-file:{self.path.relative_to(ROOT_DIR)}>.\n\n"
134 - if self.main_file.suffix == ".py":
135 - content += f"# {self.title}\n\n"
135 include = "include" if self.main_file.suffix == ".md" else \
136 "literalinclude"
138 - content += f":::{{{include}}} {make_relative(self.main_file)}\n:::\n\n"
137 + if include == "literalinclude":
138 + content += f"# {self.title}\n\n"
139 + content += f":::{{{include}}} {make_relative(self.main_file)}\n"
140 + if include == "literalinclude":
141 + content += f":language: {self.main_file.suffix[1:]}\n"
142 + content += ":::\n\n"
143
144 if not self.other_files:
145 return content
@@ -163,14 +167,16 @@ def generate_examples():
167 description=
168 "A collection of examples demonstrating usage of vLLM.\nAll documented examples are autogenerated using <gh-file:docs/source/generate_examples.py> from examples found in <gh-file:examples>.", # noqa: E501
169 caption="Examples",
166 - maxdepth=1) # TODO change to 2 when examples start being categorised
170 + maxdepth=2)
171 + # Category indices stored in reverse order because they are inserted into
172 + # examples_index.documents at index 0 in order
173 category_indices = {
168 - "offline_inference":
174 + "other":
175 Index(
170 - path=EXAMPLE_DOC_DIR / "examples_offline_inference_index.md",
171 - title="Offline Inference",
176 + path=EXAMPLE_DOC_DIR / "examples_other_index.md",
177 + title="Other",
178 description=
173 - "Offline inference examples demonstrate how to use vLLM in an offline setting, where the model is queried for predictions in batches.", # noqa: E501
179 + "Other examples that don't strongly fit into the online or offline serving categories.", # noqa: E501
180 caption="Examples",
181 ),
182 "online_serving":
@@ -181,31 +187,30 @@ def generate_examples():
187 "Online serving examples demonstrate how to use vLLM in an online setting, where the model is queried for predictions in real-time.", # noqa: E501
188 caption="Examples",
189 ),
184 - "other":
190 + "offline_inference":
191 Index(
186 - path=EXAMPLE_DOC_DIR / "examples_other_index.md",
187 - title="Other",
192 + path=EXAMPLE_DOC_DIR / "examples_offline_inference_index.md",
193 + title="Offline Inference",
194 description=
189 - "Other examples that don't strongly fit into the online or offline serving categories.", # noqa: E501
195 + "Offline inference examples demonstrate how to use vLLM in an offline setting, where the model is queried for predictions in batches.", # noqa: E501
196 caption="Examples",
197 ),
198 }
199
200 examples = []
201 + glob_patterns = ["*.py", "*.md", "*.sh"]
202 # Find categorised examples
203 for category in category_indices:
204 category_dir = EXAMPLE_DIR / category
198 - py = category_dir.glob("*.py")
199 - md = category_dir.glob("*.md")
200 - for path in itertools.chain(py, md):
205 + globs = [category_dir.glob(pattern) for pattern in glob_patterns]
206 + for path in itertools.chain(*globs):
207 examples.append(Example(path, category))
208 # Find examples in subdirectories
209 for path in category_dir.glob("*/*.md"):
210 examples.append(Example(path.parent, category))
211 # Find uncategorised examples
206 - py = EXAMPLE_DIR.glob("*.py")
207 - md = EXAMPLE_DIR.glob("*.md")
208 - for path in itertools.chain(py, md):
212 + globs = [EXAMPLE_DIR.glob(pattern) for pattern in glob_patterns]
213 + for path in itertools.chain(*globs):
214 examples.append(Example(path))
215 # Find examples in subdirectories
216 for path in EXAMPLE_DIR.glob("*/*.md"):
@@ -215,7 +220,7 @@ def generate_examples():
220 examples.append(Example(path.parent))
221
222 # Generate the example documentation
218 - for example in examples:
223 + for example in sorted(examples, key=lambda e: e.path.stem):
224 doc_path = EXAMPLE_DOC_DIR / f"{example.path.stem}.md"
225 with open(doc_path, "w+") as f:
226 f.write(example.generate())
docs/source/getting_started/installation/cpu-x86.md
+2 -2
@@ -95,7 +95,7 @@ $ VLLM_TARGET_DEVICE=cpu python setup.py install
95 $ sudo apt-get install libtcmalloc-minimal4 # install TCMalloc library
96 $ find / -name *libtcmalloc* # find the dynamic link library path
97 $ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD # prepend the library to LD_PRELOAD
98 -$ python examples/offline_inference.py # run vLLM
98 +$ python examples/offline_inference/offline_inference.py # run vLLM
99 ```
100
101 - When using the online serving, it is recommended to reserve 1-2 CPU cores for the serving framework to avoid CPU oversubscription. For example, on a platform with 32 physical CPU cores, reserving CPU 30 and 31 for the framework and using CPU 0-29 for OpenMP:
@@ -132,7 +132,7 @@ CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ MHZ
132
133 # On this platform, it is recommend to only bind openMP threads on logical CPU cores 0-7 or 8-15
134 $ export VLLM_CPU_OMP_THREADS_BIND=0-7
135 -$ python examples/offline_inference.py
135 +$ python examples/offline_inference/offline_inference.py
136 ```
137
138 - If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores using `VLLM_CPU_OMP_THREADS_BIND` to avoid cross NUMA node memory access.
docs/source/getting_started/installation/xpu.md
+1 -1
@@ -71,4 +71,4 @@ $ --pipeline-parallel-size=2 \
71 $ -tp=8
72 ```
73
74 -By default, a ray instance will be launched automatically if no existing one is detected in system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring to the <gh-file:examples/run_cluster.sh> helper script.
74 +By default, a ray instance will be launched automatically if no existing one is detected in system, with `num-gpus` equals to `parallel_config.world_size`. We recommend properly starting a ray cluster before execution, referring to the <gh-file:examples/online_serving/run_cluster.sh> helper script.
docs/source/getting_started/quickstart.md
+2 -2
@@ -31,7 +31,7 @@ For non-CUDA platforms, please refer [here](#installation-index) for specific in
31
32 ## Offline Batched Inference
33
34 -With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: <gh-file:examples/offline_inference.py>
34 +With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). See the example script: <gh-file:examples/offline_inference/offline_inference.py>
35
36 The first line of this example imports the classes {class}`~vllm.LLM` and {class}`~vllm.SamplingParams`:
37
@@ -133,7 +133,7 @@ completion = client.completions.create(model="Qwen/Qwen2.5-1.5B-Instruct",
133 print("Completion result:", completion)
134 ```
135
136 -A more detailed client example can be found here: <gh-file:examples/openai_completion_client.py>
136 +A more detailed client example can be found here: <gh-file:examples/online_serving/openai_completion_client.py>
137
138 ### OpenAI Chat Completions API with vLLM
139
docs/source/getting_started/troubleshooting.md
+1 -1
@@ -24,7 +24,7 @@ To isolate the model downloading and loading issue, you can use the `--load-form
24
25 ## Model is too large
26
27 -If the model is too large to fit in a single GPU, you might want to [consider tensor parallelism](#distributed-serving) to split the model across multiple GPUs. In that case, every process will read the whole model and split it into chunks, which makes the disk reading time even longer (proportional to the size of tensor parallelism). You can convert the model checkpoint to a sharded checkpoint using <gh-file:examples/save_sharded_state.py>. The conversion process might take some time, but later you can load the sharded checkpoint much faster. The model loading time should remain constant regardless of the size of tensor parallelism.
27 +If the model is too large to fit in a single GPU, you might want to [consider tensor parallelism](#distributed-serving) to split the model across multiple GPUs. In that case, every process will read the whole model and split it into chunks, which makes the disk reading time even longer (proportional to the size of tensor parallelism). You can convert the model checkpoint to a sharded checkpoint using <gh-file:examples/offline_inference/save_sharded_state.py>. The conversion process might take some time, but later you can load the sharded checkpoint much faster. The model loading time should remain constant regardless of the size of tensor parallelism.
28
29 ## Enable more logging
30
docs/source/models/extensions/tensorizer.md
+1 -1
@@ -9,7 +9,7 @@ shorter Pod startup times and CPU memory usage. Tensor encryption is also suppor
9
10 For more information on CoreWeave's Tensorizer, please refer to
11 [CoreWeave's Tensorizer documentation](https://github.com/coreweave/tensorizer). For more information on serializing a vLLM model, as well a general usage guide to using Tensorizer with vLLM, see
12 -the [vLLM example script](https://docs.vllm.ai/en/stable/getting_started/examples/tensorize_vllm_model.html).
12 +the [vLLM example script](https://docs.vllm.ai/en/stable/getting_started/examples/offline_inference/tensorize_vllm_model.html).
13
14 ```{note}
15 Note that to use this feature you will need to install `tensorizer` by running `pip install vllm[tensorizer]`.
docs/source/models/generative_models.md
+2 -2
@@ -46,7 +46,7 @@ for output in outputs:
46 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
47 ```
48
49 -A code example can be found here: <gh-file:examples/offline_inference.py>
49 +A code example can be found here: <gh-file:examples/offline_inference/offline_inference.py>
50
51 ### `LLM.beam_search`
52
@@ -103,7 +103,7 @@ for output in outputs:
103 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
104 ```
105
106 -A code example can be found here: <gh-file:examples/offline_inference_chat.py>
106 +A code example can be found here: <gh-file:examples/offline_inference/offline_inference_chat.py>
107
108 If the model doesn't have a chat template or you want to specify another one,
109 you can explicitly pass a chat template:
docs/source/models/pooling_models.md
+3 -3
@@ -65,7 +65,7 @@ embeds = output.outputs.embedding
65 print(f"Embeddings: {embeds!r} (size={len(embeds)})")
66 ```
67
68 -A code example can be found here: <gh-file:examples/offline_inference_embedding.py>
68 +A code example can be found here: <gh-file:examples/offline_inference/offline_inference_embedding.py>
69
70 ### `LLM.classify`
71
@@ -80,7 +80,7 @@ probs = output.outputs.probs
80 print(f"Class Probabilities: {probs!r} (size={len(probs)})")
81 ```
82
83 -A code example can be found here: <gh-file:examples/offline_inference_classification.py>
83 +A code example can be found here: <gh-file:examples/offline_inference/offline_inference_classification.py>
84
85 ### `LLM.score`
86
@@ -102,7 +102,7 @@ score = output.outputs.score
102 print(f"Score: {score}")
103 ```
104
105 -A code example can be found here: <gh-file:examples/offline_inference_scoring.py>
105 +A code example can be found here: <gh-file:examples/offline_inference/offline_inference_scoring.py>
106
107 ## Online Inference
108
docs/source/serving/distributed_serving.md
+1 -1
@@ -51,7 +51,7 @@ $ --pipeline-parallel-size 2
51
52 If a single node does not have enough GPUs to hold the model, you can run the model using multiple nodes. It is important to make sure the execution environment is the same on all nodes, including the model path, the Python environment. The recommended way is to use docker images to ensure the same environment, and hide the heterogeneity of the host machines via mapping them into the same docker configuration.
53
54 -The first step, is to start containers and organize them into a cluster. We have provided the helper script <gh-file:examples/run_cluster.sh> to start the cluster. Please note, this script launches docker without administrative privileges that would be required to access GPU performance counters when running profiling and tracing tools. For that purpose, the script can have `CAP_SYS_ADMIN` to the docker container by using the `--cap-add` option in the docker run command.
54 +The first step, is to start containers and organize them into a cluster. We have provided the helper script <gh-file:examples/online_serving/run_cluster.sh> to start the cluster. Please note, this script launches docker without administrative privileges that would be required to access GPU performance counters when running profiling and tracing tools. For that purpose, the script can have `CAP_SYS_ADMIN` to the docker container by using the `--cap-add` option in the docker run command.
55
56 Pick a node as the head node, and run the following command:
57
docs/source/serving/multimodal_inputs.md
+8 -8
@@ -60,7 +60,7 @@ for o in outputs:
60 print(generated_text)
61 ```
62
63 -Full example: <gh-file:examples/offline_inference_vision_language.py>
63 +Full example: <gh-file:examples/offline_inference/offline_inference_vision_language.py>
64
65 To substitute multiple images inside the same text prompt, you can pass in a list of images instead:
66
@@ -91,7 +91,7 @@ for o in outputs:
91 print(generated_text)
92 ```
93
94 -Full example: <gh-file:examples/offline_inference_vision_language_multi_image.py>
94 +Full example: <gh-file:examples/offline_inference/offline_inference_vision_language_multi_image.py>
95
96 Multi-image input can be extended to perform video captioning. We show this with [Qwen2-VL](https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct) as it supports videos:
97
@@ -125,13 +125,13 @@ for o in outputs:
125 You can pass a list of NumPy arrays directly to the `'video'` field of the multi-modal dictionary
126 instead of using multi-image input.
127
128 -Full example: <gh-file:examples/offline_inference_vision_language.py>
128 +Full example: <gh-file:examples/offline_inference/offline_inference_vision_language.py>
129
130 ### Audio
131
132 You can pass a tuple `(array, sampling_rate)` to the `'audio'` field of the multi-modal dictionary.
133
134 -Full example: <gh-file:examples/offline_inference_audio_language.py>
134 +Full example: <gh-file:examples/offline_inference/offline_inference_audio_language.py>
135
136 ### Embedding
137
@@ -271,7 +271,7 @@ chat_response = client.chat.completions.create(
271 print("Chat completion output:", chat_response.choices[0].message.content)
272 ```
273
274 -Full example: <gh-file:examples/openai_chat_completion_client_for_multimodal.py>
274 +Full example: <gh-file:examples/online_serving/openai_chat_completion_client_for_multimodal.py>
275
276 ```{tip}
277 Loading from local file paths is also supported on vLLM: You can specify the allowed local media path via `--allowed-local-media-path` when launching the API server/engine,
@@ -342,7 +342,7 @@ result = chat_completion_from_url.choices[0].message.content
342 print("Chat completion output from image url:", result)
343 ```
344
345 -Full example: <gh-file:examples/openai_chat_completion_client_for_multimodal.py>
345 +Full example: <gh-file:examples/online_serving/openai_chat_completion_client_for_multimodal.py>
346
347 ````{note}
348 By default, the timeout for fetching videos through HTTP URL is `30` seconds.
@@ -445,7 +445,7 @@ result = chat_completion_from_url.choices[0].message.content
445 print("Chat completion output from audio url:", result)
446 ```
447
448 -Full example: <gh-file:examples/openai_chat_completion_client_for_multimodal.py>
448 +Full example: <gh-file:examples/online_serving/openai_chat_completion_client_for_multimodal.py>
449
450 ````{note}
451 By default, the timeout for fetching audios through HTTP URL is `10` seconds.
@@ -529,4 +529,4 @@ Also important, `MrLight/dse-qwen2-2b-mrl-v1` requires a placeholder image of th
529 example below for details.
530 ```
531
532 -Full example: <gh-file:examples/openai_chat_embedding_client_for_multimodal.py>
532 +Full example: <gh-file:examples/online_serving/openai_chat_embedding_client_for_multimodal.py>
docs/source/serving/openai_compatible_server.md
+5 -5
@@ -191,7 +191,7 @@ The order of priorities is `command line > config file values > defaults`.
191 Our Completions API is compatible with [OpenAI's Completions API](https://platform.openai.com/docs/api-reference/completions);
192 you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
193
194 -Code example: <gh-file:examples/openai_completion_client.py>
194 +Code example: <gh-file:examples/online_serving/openai_completion_client.py>
195
196 #### Extra parameters
197
@@ -222,7 +222,7 @@ We support both [Vision](https://platform.openai.com/docs/guides/vision)- and
222 see our [Multimodal Inputs](#multimodal-inputs) guide for more information.
223 - *Note: `image_url.detail` parameter is not supported.*
224
225 -Code example: <gh-file:examples/openai_chat_completion_client.py>
225 +Code example: <gh-file:examples/online_serving/openai_chat_completion_client.py>
226
227 #### Extra parameters
228
@@ -255,7 +255,7 @@ which will be treated as a single prompt to the model.
255 This enables multi-modal inputs to be passed to embedding models, see [this page](#multimodal-inputs) for details.
256 ```
257
258 -Code example: <gh-file:examples/openai_embedding_client.py>
258 +Code example: <gh-file:examples/online_serving/openai_embedding_client.py>
259
260 #### Extra parameters
261
@@ -299,7 +299,7 @@ Our Pooling API encodes input prompts using a [pooling model](../models/pooling_
299
300 The input format is the same as [Embeddings API](#embeddings-api), but the output data can contain an arbitrary nested list, not just a 1-D list of floats.
301
302 -Code example: <gh-file:examples/openai_pooling_client.py>
302 +Code example: <gh-file:examples/online_serving/openai_pooling_client.py>
303
304 (score-api)=
305 ### Score API
@@ -309,7 +309,7 @@ Usually, the score for a sentence pair refers to the similarity between two sent
309
310 You can find the documentation for these kind of models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
311
312 -Code example: <gh-file:examples/openai_cross_encoder_score.py>
312 +Code example: <gh-file:examples/online_serving/openai_cross_encoder_score.py>
313
314 #### Single inference
315
examples/offline_inference/aqlm_example.py renamed
examples/offline_inference/cpu_offload.py renamed
examples/offline_inference/florence2_inference.py renamed
+2 -1
@@ -3,7 +3,8 @@ Demonstrate prompting of text-to-text
3 encoder/decoder models, specifically Florence-2
4 '''
5 # TODO(Isotr0py):
6 -# Move to offline_inference_vision_language.py after porting vision backbone
6 +# Move to offline_inference/offline_inference_vision_language.py
7 +# after porting vision backbone
8 from vllm import LLM, SamplingParams
9
10 dtype = "float"
examples/offline_inference/gguf_inference.py renamed
examples/offline_inference/llm_engine_example.py renamed
examples/offline_inference/lora_with_quantization_inference.py renamed
examples/offline_inference/multilora_inference.py renamed
examples/offline_inference/offline_chat_with_tools.py renamed
examples/offline_inference/offline_inference.py renamed
examples/offline_inference/offline_inference_arctic.py renamed
examples/offline_inference/offline_inference_audio_language.py renamed
examples/offline_inference/offline_inference_chat.py renamed
examples/offline_inference/offline_inference_classification.py renamed
examples/offline_inference/offline_inference_cli.py renamed
examples/offline_inference/offline_inference_distributed.py renamed
examples/offline_inference/offline_inference_embedding.py renamed
examples/offline_inference/offline_inference_encoder_decoder.py renamed
examples/offline_inference/offline_inference_mlpspeculator.py renamed
examples/offline_inference/offline_inference_neuron.py renamed
examples/offline_inference/offline_inference_neuron_int8_quantization.py renamed
examples/offline_inference/offline_inference_openai/offline_inference_openai.md renamed
+9 -9
@@ -8,7 +8,7 @@ This is a guide to performing batch inference using the OpenAI batch file format
8
9 The OpenAI batch file format consists of a series of json objects on new lines.
10
11 -[See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/openai_example_batch.jsonl)
11 +[See here for an example file.](https://github.com/vllm-project/vllm/blob/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl)
12
13 Each line represents a separate request. See the [OpenAI package reference](https://platform.openai.com/docs/api-reference/batch/requestInput) for more details.
14
@@ -31,13 +31,13 @@ We currently only support `/v1/chat/completions` and `/v1/embeddings` endpoints
31 To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
32
33 ```
34 -wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl
34 +wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl
35 ```
36
37 Once you've created your batch file it should look like this
38
39 ```
40 -$ cat openai_example_batch.jsonl
40 +$ cat offline_inference/offline_inference_openai/openai_example_batch.jsonl
41 {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
42 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
43 ```
@@ -49,7 +49,7 @@ The batch running tool is designed to be used from the command line.
49 You can run the batch with the following command, which will write its results to a file called `results.jsonl`
50
51 ```
52 -python -m vllm.entrypoints.openai.run_batch -i openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
52 +python -m vllm.entrypoints.openai.run_batch -i offline_inference/offline_inference_openai/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
53 ```
54
55 ### Step 3: Check your results
@@ -66,10 +66,10 @@ $ cat results.jsonl
66
67 The batch runner supports remote input and output urls that are accessible via http/https.
68
69 -For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl`, you can run
69 +For example, to run against our example input file located at `https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl`, you can run
70
71 ```
72 -python -m vllm.entrypoints.openai.run_batch -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
72 +python -m vllm.entrypoints.openai.run_batch -i https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl -o results.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct
73 ```
74
75 ## Example 3: Integrating with AWS S3
@@ -90,13 +90,13 @@ To integrate with cloud blob storage, we recommend using presigned urls.
90 To follow along with this example, you can download the example batch, or create your own batch file in your working directory.
91
92 ```
93 -wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/openai_example_batch.jsonl
93 +wget https://raw.githubusercontent.com/vllm-project/vllm/main/examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl
94 ```
95
96 Once you've created your batch file it should look like this
97
98 ```
99 -$ cat openai_example_batch.jsonl
99 +$ cat offline_inference/offline_inference_openai/openai_example_batch.jsonl
100 {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
101 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "meta-llama/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_completion_tokens": 1000}}
102 ```
@@ -104,7 +104,7 @@ $ cat openai_example_batch.jsonl
104 Now upload your batch file to your S3 bucket.
105
106 ```
107 -aws s3 cp openai_example_batch.jsonl s3://MY_BUCKET/MY_INPUT_FILE.jsonl
107 +aws s3 cp offline_inference/offline_inference_openai/openai_example_batch.jsonl s3://MY_BUCKET/MY_INPUT_FILE.jsonl
108 ```
109
110 ### Step 2: Generate your presigned urls
examples/offline_inference/offline_inference_openai/openai_example_batch.jsonl renamed
examples/offline_inference/offline_inference_pixtral.py renamed
examples/offline_inference/offline_inference_scoring.py renamed
examples/offline_inference/offline_inference_structured_outputs.py renamed
examples/offline_inference/offline_inference_tpu.py renamed
examples/offline_inference/offline_inference_vision_language.py renamed
examples/offline_inference/offline_inference_vision_language_embedding.py renamed
examples/offline_inference/offline_inference_vision_language_multi_image.py renamed
examples/offline_inference/offline_inference_whisper.py renamed
examples/offline_inference/offline_inference_with_default_generation_config.py renamed
examples/offline_inference/offline_inference_with_prefix.py renamed
examples/offline_inference/offline_inference_with_profiler.py renamed
examples/offline_inference/offline_profile.py renamed
+1 -1
@@ -363,7 +363,7 @@ Profile a model
363
364 example:
365 ```
366 - python examples/offline_profile.py \\
366 + python examples/offline_inference/offline_profile.py \\
367 --model neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 --batch-size 4 \\
368 --prompt-len 512 --max-num-batched-tokens 8196 --json Llama31-8b-FP8 \\
369 --enforce-eager run_num_steps -n 2
examples/offline_inference/save_sharded_state.py renamed
examples/online_serving/api_client.py renamed
examples/online_serving/chart-helm/.helmignore renamed
examples/online_serving/chart-helm/Chart.yaml renamed
examples/online_serving/chart-helm/README.md new
+21
@@ -0,0 +1,21 @@
1 +# Helm Charts
2 +
3 +This directory contains a Helm chart for deploying the vllm application. The chart includes configurations for deployment, autoscaling, resource management, and more.
4 +
5 +## Files
6 +
7 +- Chart.yaml: Defines the chart metadata including name, version, and maintainers.
8 +- ct.yaml: Configuration for chart testing.
9 +- lintconf.yaml: Linting rules for YAML files.
10 +- values.schema.json: JSON schema for validating values.yaml.
11 +- values.yaml: Default values for the Helm chart.
12 +- templates/_helpers.tpl: Helper templates for defining common configurations.
13 +- templates/configmap.yaml: Template for creating ConfigMaps.
14 +- templates/custom-objects.yaml: Template for custom Kubernetes objects.
15 +- templates/deployment.yaml: Template for creating Deployments.
16 +- templates/hpa.yaml: Template for Horizontal Pod Autoscaler.
17 +- templates/job.yaml: Template for Kubernetes Jobs.
18 +- templates/poddisruptionbudget.yaml: Template for Pod Disruption Budget.
19 +- templates/pvc.yaml: Template for Persistent Volume Claims.
20 +- templates/secrets.yaml: Template for Kubernetes Secrets.
21 +- templates/service.yaml: Template for creating Services.
\ No newline at end of file
examples/online_serving/chart-helm/ct.yaml renamed
examples/online_serving/chart-helm/lintconf.yaml renamed
examples/online_serving/chart-helm/templates/_helpers.tpl renamed
examples/online_serving/chart-helm/templates/configmap.yaml renamed
examples/online_serving/chart-helm/templates/custom-objects.yaml renamed
examples/online_serving/chart-helm/templates/deployment.yaml renamed
examples/online_serving/chart-helm/templates/hpa.yaml renamed
examples/online_serving/chart-helm/templates/job.yaml renamed
examples/online_serving/chart-helm/templates/poddisruptionbudget.yaml renamed
examples/online_serving/chart-helm/templates/pvc.yaml renamed
examples/online_serving/chart-helm/templates/secrets.yaml renamed
examples/online_serving/chart-helm/templates/service.yaml renamed
examples/online_serving/chart-helm/values.schema.json renamed
examples/online_serving/chart-helm/values.yaml renamed
examples/online_serving/disaggregated_prefill.sh renamed
examples/online_serving/gradio_openai_chatbot_webserver.py renamed
examples/online_serving/gradio_webserver.py renamed
examples/online_serving/openai_chat_completion_client.py renamed
examples/online_serving/openai_chat_completion_client_for_multimodal.py renamed
examples/online_serving/openai_chat_completion_client_with_tools.py renamed
examples/online_serving/openai_chat_completion_structured_outputs.py renamed
examples/online_serving/openai_chat_embedding_client_for_multimodal.py renamed
examples/online_serving/openai_completion_client.py renamed
examples/online_serving/openai_cross_encoder_score.py renamed
examples/online_serving/openai_embedding_client.py renamed
examples/online_serving/openai_pooling_client.py renamed
examples/online_serving/opentelemetry/Otel.md renamed
examples/online_serving/opentelemetry/dummy_client.py renamed
examples/online_serving/prometheus_grafana/README.md renamed
examples/online_serving/prometheus_grafana/docker-compose.yaml renamed
examples/online_serving/prometheus_grafana/grafana.json renamed
examples/online_serving/prometheus_grafana/prometheus.yaml renamed
examples/online_serving/run_cluster.sh renamed
examples/online_serving/sagemaker-entrypoint.sh renamed
examples/other/fp8/README.md renamed
+5 -5
@@ -20,12 +20,12 @@ Before incorporating the FP8 datatype for inference workloads, you must adhere t
20 ### 2. Convert HF model into a quantized HF model.
21 Note: The following steps are adapted from the [TensorRT-LLM repository](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/quantization/README.md).
22
23 -`quantize.py` (examples/fp8/quantizer/quantize.py) uses the quantization toolkit (AMMO) to calibrate the PyTorch models and export TensorRT-LLM checkpoints. Each TensorRT-LLM checkpoint contains a config file (in .json format) and one or several rank weight files (in .safetensors format).
23 +`quantize.py` (examples/other/fp8/quantizer/quantize.py) uses the quantization toolkit (AMMO) to calibrate the PyTorch models and export TensorRT-LLM checkpoints. Each TensorRT-LLM checkpoint contains a config file (in .json format) and one or several rank weight files (in .safetensors format).
24
25 -The detailed quantization toolkit (AMMO) conversion guide for FP8 can be found at `examples/fp8/quantizer/README.md`.
25 +The detailed quantization toolkit (AMMO) conversion guide for FP8 can be found at `examples/other/fp8/quantizer/README.md`.
26
27 ### 3. Extract KV Cache Scaling Factors from quantized HF model.
28 -`extract_scales.py` (examples/fp8/extract_scales.py) can be utilized to extract the KV cache scaling factors from your quantized HF model, however at the moment, this tool exclusively supports Llama 2 models. It is also important to note the following:
28 +`extract_scales.py` (examples/other/fp8/extract_scales.py) can be utilized to extract the KV cache scaling factors from your quantized HF model, however at the moment, this tool exclusively supports Llama 2 models. It is also important to note the following:
29 1. **File Structure**: The utility operates under the assumption that all parameters, including KV cache scaling factors, corresponding to a particular Tensor Parallelism (TP) rank are stored in a single file. These files must adhere to a specific naming convention where the TP rank is immediately identified after a specific keyword (e.g., "rank") in the filename.
30
31 2. **TP Decomposition**: The utility assumes consistency between the TP decomposition employed by the quantizer tool and that used by vLLM.
@@ -35,7 +35,7 @@ The detailed quantization toolkit (AMMO) conversion guide for FP8 can be found a
35 ```python
36 # prerequisites:
37 # - Quantized HF LLaMa 2 model
38 -python3 examples/fp8/extract_scales.py --help
38 +python3 examples/other/fp8/extract_scales.py --help
39 Usage: extract_scales.py [-h] --quantized_model QUANTIZED_MODEL [--load_format {auto,safetensors,npz,pt}] [--output_dir OUTPUT_DIR] [--output_name OUTPUT_NAME] [--tp_size TP_SIZE]
40
41 KV Scale Extraction Example
@@ -52,7 +52,7 @@ Optional arguments:
52 ```
53 ```python
54 Example:
55 -python3 examples/fp8/extract_scales.py --quantized_model <QUANTIZED_MODEL_DIR> --tp_size <TENSOR_PARALLEL_SIZE> --output_dir <PATH_TO_OUTPUT_DIR>
55 +python3 examples/other/fp8/extract_scales.py --quantized_model <QUANTIZED_MODEL_DIR> --tp_size <TENSOR_PARALLEL_SIZE> --output_dir <PATH_TO_OUTPUT_DIR>
56 ```
57 ### 4. Load KV Cache Scaling Factors into VLLM.
58 This script evaluates the inference throughput of language models using various backends such as vLLM. It measures the time taken to process a given number of prompts and generate sequences for each prompt. The recently generated KV cache scaling factors are now integrated into the benchmarking process and allow for KV cache scaling factors to be utilized for FP8.
examples/other/fp8/extract_scales.py renamed
examples/other/fp8/quantizer/README.md renamed
examples/other/fp8/quantizer/quantize.py renamed
examples/other/logging_configuration.md renamed
examples/other/tensorize_vllm_model.py renamed
+5 -5
@@ -25,7 +25,7 @@ https://github.com/coreweave/tensorizer
25 To serialize a model, install vLLM from source, then run something
26 like this from the root level of this repository:
27
28 -python -m examples.tensorize_vllm_model \
28 +python -m examples.offline_inference.tensorize_vllm_model \
29 --model facebook/opt-125m \
30 serialize \
31 --serialized-directory s3://my-bucket \
@@ -45,7 +45,7 @@ providing a `--keyfile` argument.
45 To deserialize a model, you can run something like this from the root
46 level of this repository:
47
48 -python -m examples.tensorize_vllm_model \
48 +python -m examples.offline_inference.tensorize_vllm_model \
49 --model EleutherAI/gpt-j-6B \
50 --dtype float16 \
51 deserialize \
@@ -63,11 +63,11 @@ shard's rank. Sharded models serialized with this script will be named as
63 model-rank-%03d.tensors
64
65 For more information on the available arguments for serializing, run
66 -`python -m examples.tensorize_vllm_model serialize --help`.
66 +`python -m examples.offline_inference.tensorize_vllm_model serialize --help`.
67
68 Or for deserializing:
69
70 -`python -m examples.tensorize_vllm_model deserialize --help`.
70 +`python -m examples.offline_inference.tensorize_vllm_model deserialize --help`.
71
72 Once a model is serialized, tensorizer can be invoked with the `LLM` class
73 directly to load models:
@@ -88,7 +88,7 @@ TensorizerConfig arguments desired.
88 In order to see all of the available arguments usable to configure
89 loading with tensorizer that are given to `TensorizerConfig`, run:
90
91 -`python -m examples.tensorize_vllm_model deserialize --help`
91 +`python -m examples.offline_inference.tensorize_vllm_model deserialize --help`
92
93 under the `tensorizer options` section. These can also be used for
94 deserialization in this example script, although `--tensorizer-uri` and
pyproject.toml
+1 -1
@@ -20,7 +20,7 @@ build-backend = "setuptools.build_meta"
20 line-length = 80
21 exclude = [
22 # External file, leaving license intact
23 - "examples/fp8/quantizer/quantize.py"
23 + "examples/other/fp8/quantizer/quantize.py"
24 ]
25
26 [tool.ruff.lint.per-file-ignores]
tests/plugins_tests/test_platform_plugins.py
+1 -1
@@ -5,7 +5,7 @@ def test_platform_plugins():
5 import os
6 example_file = os.path.join(
7 os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
8 - "examples", "offline_inference.py")
8 + "examples", "offline_inference/offline_inference.py")
9 runpy.run_path(example_file)
10
11 # check if the plugin is loaded correctly
tests/tensorizer_loader/test_tensorizer.py
+2 -2
@@ -163,8 +163,8 @@ def test_deserialized_hf_model_has_same_outputs(hf_runner, vllm_runner,
163
164 def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
165 multilora_inference = import_from_path(
166 - "examples.multilora_inference",
167 - EXAMPLES_PATH / "multilora_inference.py",
166 + "examples.offline_inference.multilora_inference",
167 + EXAMPLES_PATH / "offline_inference/multilora_inference.py",
168 )
169
170 model_ref = "meta-llama/Llama-2-7b-hf"
tools/profiler/print_layerwise_table.py
+1 -1
@@ -31,7 +31,7 @@ if __name__ == "__main__":
31 type=str,
32 required=True,
33 help="json trace file output by "
34 - "examples/offline_profile.py")
34 + "examples/offline_inference/offline_profile.py")
35 parser.add_argument("--phase",
36 type=str,
37 required=True,
tools/profiler/visualize_layerwise_profile.py
+5 -5
@@ -534,11 +534,11 @@ def main(
534 if __name__ == "__main__":
535 parser = argparse.ArgumentParser()
536
537 - parser.add_argument(
538 - "--json-trace",
539 - type=str,
540 - required=True,
541 - help="json trace file output by examples/offline_profile.py")
537 + parser.add_argument("--json-trace",
538 + type=str,
539 + required=True,
540 + help="json trace file output by \
541 + examples/offline_inference/offline_profile.py")
542 parser.add_argument("--output-directory",
543 type=str,
544 required=False,
vllm/distributed/kv_transfer/README.md
+1 -1
@@ -22,7 +22,7 @@ NOTE: If you want to not only transfer KV caches, but adjust the model execution
22
23 ## Disaggregated prefilling
24
25 -The example usage is in [this file](../../../examples/disaggregated_prefill.sh).
25 +The example usage is in [this file](../../../examples/online_serving/disaggregated_prefill.sh).
26
27 Here is the diagram of how we run disaggretgated prefilling.
28
vllm/model_executor/model_loader/loader.py
+6 -5
@@ -452,9 +452,9 @@ class TensorizerLoader(BaseModelLoader):
452 """Load a serialized model with tensorizer to the CPU.
453
454 This is only necessary when the model isn't vLLM-tensorized (see
455 - examples/tensorize_vllm_model.py) This should still be faster than
456 - default HuggingFace loading, but will be slower than loading a
457 - vLLM-tensorized model.
455 + examples/other/tensorize_vllm_model.py) This should still
456 + be faster than default HuggingFace loading, but will be slower than
457 + loading a vLLM-tensorized model.
458 """
459 device_config = vllm_config.device_config
460 model_config = vllm_config.model_config
@@ -472,7 +472,7 @@ class TensorizerLoader(BaseModelLoader):
472 """Load a serialized model with tensorizer.
473
474 Expects a vLLM-tensorized model. See the
475 - examples/tensorize_vllm_model.py example script
475 + examples/other/tensorize_vllm_model.py example script
476 for serializing vLLM models."""
477
478 device_config = vllm_config.device_config
@@ -529,7 +529,8 @@ class ShardedStateLoader(BaseModelLoader):
529 Model loader that directly loads each worker's model state dict, which
530 enables a fast load path for large tensor-parallel models where each worker
531 only needs to read its own shard rather than the entire checkpoint. See
532 - `examples/save_sharded_state.py` for creating a sharded checkpoint.
532 + `examples/offline_inference/save_sharded_state.py` for creating a sharded
533 + checkpoint.
534 """
535
536 DEFAULT_PATTERN = "model-rank-{rank}-part-{part}.safetensors"
vllm/model_executor/model_loader/tensorizer.py
+7 -7
@@ -155,7 +155,7 @@ class TensorizerArgs:
155 encryption_keyfile: File path to a binary file containing a
156 binary key to use for decryption. `None` (the default) means
157 no decryption. See the example script in
158 - examples/tensorize_vllm_model.py.
158 + examples/other/tensorize_vllm_model.py.
159 s3_access_key_id: The access key for the S3 bucket. Can also be set via
160 the S3_ACCESS_KEY_ID environment variable.
161 s3_secret_access_key: The secret access key for the S3 bucket. Can also
@@ -363,12 +363,12 @@ class TensorizerAgent:
363 def tensorizer_weights_iterator(
364 tensorizer_args: "TensorizerArgs"
365 ) -> Generator[Tuple[str, torch.Tensor], None, None]:
366 - logger.warning(
367 - "Deserializing HuggingFace models is not optimized for "
368 - "loading on vLLM, as tensorizer is forced to load to CPU. "
369 - "Consider deserializing a vLLM model instead for faster "
370 - "load times. See the examples/tensorize_vllm_model.py example "
371 - "script for serializing vLLM models.")
366 + logger.warning("Deserializing HuggingFace models is not optimized for "
367 + "loading on vLLM, as tensorizer is forced to load to CPU. "
368 + "Consider deserializing a vLLM model instead for faster "
369 + "load times. See the "
370 + "examples/other/tensorize_vllm_model.py example script "
371 + "for serializing vLLM models.")
372
373 deserializer_args = tensorizer_args.deserializer_params
374 stream_params = tensorizer_args.stream_params
vllm/model_executor/model_loader/weight_utils.py
+2 -1
@@ -503,7 +503,8 @@ def kv_cache_scales_loader(
503 KV cache scaling factors. The serialization should represent a dictionary
504 whose keys are the TP ranks and values are another dictionary mapping layers
505 to their KV cache scaling factors.
506 - Keep this function in sync with the output of examples/fp8/extract_scales.py
506 + Keep this function in sync with the output of
507 + examples/other/fp8/extract_scales.py
508 """
509 try:
510 with open(filename) as f: