@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4abf6336ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add one example to run batch inference distributed on Ray (#2696)

Cheng Su committed Feb 2, 2024 at 15:41 UTC 4abf6336ec65c270343eb895e7b18786e9274176
1 file changed +70
examples/offline_inference_distributed.py new
+70
@@ -0,0 +1,70 @@
1 +"""
2 +This example shows how to use Ray Data for running offline batch inference
3 +distributively on a multi-nodes cluster.
4 +
5 +Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html
6 +"""
7 +
8 +from vllm import LLM, SamplingParams
9 +from typing import Dict
10 +import numpy as np
11 +import ray
12 +
13 +# Create a sampling params object.
14 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
15 +
16 +
17 +# Create a class to do batch inference.
18 +class LLMPredictor:
19 +
20 + def __init__(self):
21 + # Create an LLM.
22 + self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf")
23 +
24 + def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]:
25 + # Generate texts from the prompts.
26 + # The output is a list of RequestOutput objects that contain the prompt,
27 + # generated text, and other information.
28 + outputs = self.llm.generate(batch["text"], sampling_params)
29 + prompt = []
30 + generated_text = []
31 + for output in outputs:
32 + prompt.append(output.prompt)
33 + generated_text.append(' '.join([o.text for o in output.outputs]))
34 + return {
35 + "prompt": prompt,
36 + "generated_text": generated_text,
37 + }
38 +
39 +
40 +# Read one text file from S3. Ray Data supports reading multiple files
41 +# from cloud storage (such as JSONL, Parquet, CSV, binary format).
42 +ds = ray.data.read_text("s3://anonymous@air-example-data/prompts.txt")
43 +
44 +# Apply batch inference for all input data.
45 +ds = ds.map_batches(
46 + LLMPredictor,
47 + # Set the concurrency to the number of LLM instances.
48 + concurrency=10,
49 + # Specify the number of GPUs required per LLM instance.
50 + # NOTE: Do NOT set `num_gpus` when using vLLM with tensor-parallelism
51 + # (i.e., `tensor_parallel_size`).
52 + num_gpus=1,
53 + # Specify the batch size for inference.
54 + batch_size=32,
55 +)
56 +
57 +# Peek first 10 results.
58 +# NOTE: This is for local testing and debugging. For production use case,
59 +# one should write full result out as shown below.
60 +outputs = ds.take(limit=10)
61 +for output in outputs:
62 + prompt = output["prompt"]
63 + generated_text = output["generated_text"]
64 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
65 +
66 +# Write inference output data out as Parquet files to S3.
67 +# Multiple files would be written to the output destination,
68 +# and each task would write one or more files separately.
69 +#
70 +# ds.write_parquet("s3://<your-output-bucket>")