@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c5711ef985
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Doc] Update Ray Data distributed offline inference example (#4871)

Antoni Baum committed May 17, 2024 at 10:52 UTC c5711ef98519de25d1f51121f7848a13f2891fc1
1 file changed +42 -6
examples/offline_inference_distributed.py
+42 -6
@@ -9,19 +9,31 @@ from typing import Dict
9
10 import numpy as np
11 import ray
12 +from packaging.version import Version
13 +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
14
15 from vllm import LLM, SamplingParams
16
17 +assert Version(ray.__version__) >= Version(
18 + "2.22.0"), "Ray version must be at least 2.22.0"
19 +
20 # Create a sampling params object.
21 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
22
23 +# Set tensor parallelism per instance.
24 +tensor_parallel_size = 1
25 +
26 +# Set number of instances. Each instance will use tensor_parallel_size GPUs.
27 +num_instances = 1
28 +
29
30 # Create a class to do batch inference.
31 class LLMPredictor:
32
33 def __init__(self):
34 # Create an LLM.
24 - self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf")
35 + self.llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",
36 + tensor_parallel_size=tensor_parallel_size)
37
38 def __call__(self, batch: Dict[str, np.ndarray]) -> Dict[str, list]:
39 # Generate texts from the prompts.
@@ -43,17 +55,41 @@ class LLMPredictor:
55 # from cloud storage (such as JSONL, Parquet, CSV, binary format).
56 ds = ray.data.read_text("s3://anonymous@air-example-data/prompts.txt")
57
58 +
59 +# For tensor_parallel_size > 1, we need to create placement groups for vLLM
60 +# to use. Every actor has to have its own placement group.
61 +def scheduling_strategy_fn():
62 + # One bundle per tensor parallel worker
63 + pg = ray.util.placement_group(
64 + [{
65 + "GPU": 1,
66 + "CPU": 1
67 + }] * tensor_parallel_size,
68 + strategy="STRICT_PACK",
69 + )
70 + return dict(scheduling_strategy=PlacementGroupSchedulingStrategy(
71 + pg, placement_group_capture_child_tasks=True))
72 +
73 +
74 +resources_kwarg = {}
75 +if tensor_parallel_size == 1:
76 + # For tensor_parallel_size == 1, we simply set num_gpus=1.
77 + resources_kwarg["num_gpus"] = 1
78 +else:
79 + # Otherwise, we have to set num_gpus=0 and provide
80 + # a function that will create a placement group for
81 + # each instance.
82 + resources_kwarg["num_gpus"] = 0
83 + resources_kwarg["ray_remote_args_fn"] = scheduling_strategy_fn
84 +
85 # Apply batch inference for all input data.
86 ds = ds.map_batches(
87 LLMPredictor,
88 # Set the concurrency to the number of LLM instances.
50 - concurrency=10,
51 - # Specify the number of GPUs required per LLM instance.
52 - # NOTE: Do NOT set `num_gpus` when using vLLM with tensor-parallelism
53 - # (i.e., `tensor_parallel_size`).
54 - num_gpus=1,
89 + concurrency=num_instances,
90 # Specify the batch size for inference.
91 batch_size=32,
92 + **resources_kwarg,
93 )
94
95 # Peek first 10 results.