A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Doc] Update Ray Data distributed offline inference example (#4871)
Antoni Baum committed
May 17, 2024 at 10:52 UTC
c5711ef98519de25d1f51121f7848a13f2891fc1
1 file changed
+42
-6
examples/offline_inference_distributed.py
+42
-6
@@ -9,19 +9,31 @@ from typing import Dict
9
10
import numpy as np
11
import ray
12
+from packaging.version import Version
13
+from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
14
15
from vllm import LLM, SamplingParams
16
17
+assert Version(ray.__version__) >= Version(
18
+ "2.22.0"), "Ray version must be at least 2.22.0"
19
+
20
# Create a sampling params object.
21
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22
23
+# Set tensor parallelism per instance.
24
+tensor_parallel_size = 1
25
+
26
+# Set number of instances. Each instance will use tensor_parallel_size GPUs.
27
+num_instances = 1
28
+
29
30
# Create a class to do batch inference.
31
class LLMPredictor:
32
33
def __init__(self):
34
# Create an LLM.
24
- self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf")
35
+ self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",
36
+ tensor_parallel_size=tensor_parallel_size)
37
38
def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]:
39
# Generate texts from the prompts.
@@ -43,17 +55,41 @@ class LLMPredictor:
55
# from cloud storage (such as JSONL, Parquet, CSV, binary format).
56
ds = ray.data.read_text("s3://anonymous@air-example-data/prompts.txt")
57
58
+
59
+# For tensor_parallel_size > 1, we need to create placement groups for vLLM
60
+# to use. Every actor has to have its own placement group.
61
+def scheduling_strategy_fn():
62
+ # One bundle per tensor parallel worker
63
+ pg = ray.util.placement_group(
64
+ [{
65
+ "GPU": 1,
66
+ "CPU": 1
67
+ }] * tensor_parallel_size,
68
+ strategy="STRICT_PACK",
69
+ )
70
+ return dict(scheduling_strategy=PlacementGroupSchedulingStrategy(
71
+ pg, placement_group_capture_child_tasks=True))
72
+
73
+
74
+resources_kwarg = {}
75
+if tensor_parallel_size == 1:
76
+ # For tensor_parallel_size == 1, we simply set num_gpus=1.
77
+ resources_kwarg["num_gpus"] = 1
78
+else:
79
+ # Otherwise, we have to set num_gpus=0 and provide
80
+ # a function that will create a placement group for
81
+ # each instance.
82
+ resources_kwarg["num_gpus"] = 0
83
+ resources_kwarg["ray_remote_args_fn"] = scheduling_strategy_fn
84
+
85
# Apply batch inference for all input data.
86
ds = ds.map_batches(
87
LLMPredictor,
88
# Set the concurrency to the number of LLM instances.
50
- concurrency=10,
51
- # Specify the number of GPUs required per LLM instance.
52
- # NOTE: Do NOT set `num_gpus` when using vLLM with tensor-parallelism
53
- # (i.e., `tensor_parallel_size`).
54
- num_gpus=1,
89
+ concurrency=num_instances,
90
# Specify the batch size for inference.
91
batch_size=32,
92
+ **resources_kwarg,
93
)
94
95
# Peek first 10 results.