A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add one example to run batch inference distributed on Ray (#2696)
Cheng Su committed
Feb 2, 2024 at 15:41 UTC
4abf6336ec65c270343eb895e7b18786e9274176
1 file changed
+70
examples/offline_inference_distributed.py
new
+70
@@ -0,0 +1,70 @@
1
+"""
2
+This example shows how to use Ray Data for running offline batch inference
3
+distributively on a multi-nodes cluster.
4
+
5
+Learn more about Ray Data in https://docs.ray.io/en/latest/data/data.html
6
+"""
7
+
8
+from vllm import LLM, SamplingParams
9
+from typing import Dict
10
+import numpy as np
11
+import ray
12
+
13
+# Create a sampling params object.
14
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
15
+
16
+
17
+# Create a class to do batch inference.
18
+class LLMPredictor:
19
+
20
+ def __init__(self):
21
+ # Create an LLM.
22
+ self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf")
23
+
24
+ def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]:
25
+ # Generate texts from the prompts.
26
+ # The output is a list of RequestOutput objects that contain the prompt,
27
+ # generated text, and other information.
28
+ outputs = self.llm.generate(batch["text"], sampling_params)
29
+ prompt = []
30
+ generated_text = []
31
+ for output in outputs:
32
+ prompt.append(output.prompt)
33
+ generated_text.append(' '.join([o.text for o in output.outputs]))
34
+ return {
35
+ "prompt": prompt,
36
+ "generated_text": generated_text,
37
+ }
38
+
39
+
40
+# Read one text file from S3. Ray Data supports reading multiple files
41
+# from cloud storage (such as JSONL, Parquet, CSV, binary format).
42
+ds = ray.data.read_text("s3://anonymous@air-example-data/prompts.txt")
43
+
44
+# Apply batch inference for all input data.
45
+ds = ds.map_batches(
46
+ LLMPredictor,
47
+ # Set the concurrency to the number of LLM instances.
48
+ concurrency=10,
49
+ # Specify the number of GPUs required per LLM instance.
50
+ # NOTE: Do NOT set `num_gpus` when using vLLM with tensor-parallelism
51
+ # (i.e., `tensor_parallel_size`).
52
+ num_gpus=1,
53
+ # Specify the batch size for inference.
54
+ batch_size=32,
55
+)
56
+
57
+# Peek first 10 results.
58
+# NOTE: This is for local testing and debugging. For production use case,
59
+# one should write full result out as shown below.
60
+outputs = ds.take(limit=10)
61
+for output in outputs:
62
+ prompt = output["prompt"]
63
+ generated_text = output["generated_text"]
64
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
65
+
66
+# Write inference output data out as Parquet files to S3.
67
+# Multiple files would be written to the output destination,
68
+# and each task would write one or more files separately.
69
+#
70
+# ds.write_parquet("s3://<your-output-bucket>")