A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add docstrings for LLM (#137)
Woosuk Kwon committed
Jun 4, 2023 at 12:52 UTC
8274ca23ac9f2ea0ccf758d1883794643aecb2e0
4 files changed
+66
-10
benchmarks/benchmark_latency.py
+2
-2
@@ -30,7 +30,6 @@ def main(args: argparse.Namespace):
30
max_tokens=args.output_len,
31
)
32
print(sampling_params)
33
- dummy_prompts = [""] * args.batch_size
33
dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
34
35
def run_to_completion(profile: bool = False):
@@ -38,7 +37,8 @@ def main(args: argparse.Namespace):
37
torch.cuda.cudart().cudaProfilerStart()
38
start_time = time.time()
39
41
- llm.generate(dummy_prompts, sampling_params, dummy_prompt_token_ids,
40
+ llm.generate(prompt_token_ids=dummy_prompt_token_ids,
41
+ sampling_params=sampling_params,
42
use_tqdm=False)
43
44
end_time = time.time()
benchmarks/benchmark_throughput.py
+5
-3
@@ -72,9 +72,9 @@ def main(args: argparse.Namespace):
72
)
73
# FIXME(woosuk): Do not use internal method.
74
llm._add_request(
75
- prompt="",
76
- sampling_params=sampling_params,
75
+ prompt=None,
76
prompt_token_ids=prompt_token_ids,
77
+ sampling_params=sampling_params,
78
)
79
80
start = time.time()
@@ -85,7 +85,9 @@ def main(args: argparse.Namespace):
85
len(prompt_token_ids) + output_len
86
for prompt_token_ids, output_len in requests
87
)
88
- print(f"Throughput: {total_num_tokens / (end - start):.2f} tokens/s")
88
+ elapsed_time = end - start
89
+ print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
90
+ f"{total_num_tokens / elapsed_time:.2f} tokens/s")
91
92
93
if __name__ == "__main__":
cacheflow/entrypoints/llm.py
+57
-4
@@ -11,6 +11,28 @@ from cacheflow.utils import Counter
11
12
13
class LLM:
14
+ """An LLM for generating texts from given prompts and sampling parameters.
15
+
16
+ This class includes a tokenizer, a language model (possibly distributed
17
+ across multiple GPUs), and GPU memory space allocated for intermediate
18
+ states (aka KV cache). Given a batch of prompts and sampling parameters,
19
+ this class generates texts from the model, using an intelligent batching
20
+ mechanism and efficient memory management.
21
+
22
+ NOTE: This class is intended to be used for offline inference. For online
23
+ serving, use the `AsyncLLMServer` class instead.
24
+ NOTE: For the comprehensive list of arguments, see `ServerArgs`.
25
+
26
+ Args:
27
+ model: The name or path of a HuggingFace Transformers model.
28
+ tensor_parallel_size: The number of GPUs to use for distributed
29
+ execution with tensor parallelism.
30
+ dtype: The data type for the model weights and activations. Currently,
31
+ we support `float16` and `bfloat16`. If `default`, we use the
32
+ `torch_dtype` attribute of the model config. If the `torch_dtype`
33
+ is `float32`, we use `float16` instead.
34
+ seed: The seed to initialize the random number generator for sampling.
35
+ """
36
37
def __init__(
38
self,
@@ -39,19 +61,50 @@ class LLM:
61
62
def generate(
63
self,
42
- prompts: Union[str, List[str]],
64
+ prompts: Optional[Union[str, List[str]]] = None,
65
sampling_params: Optional[SamplingParams] = None,
66
prompt_token_ids: Optional[List[List[int]]] = None,
67
use_tqdm: bool = True,
68
) -> List[RequestOutput]:
69
+ """Generates the completions for the input prompts.
70
+
71
+ NOTE: This class automatically batches the given prompts, considering
72
+ the memory constraint. For the best performance, put all of your prompts
73
+ into a single list and pass it to this method.
74
+
75
+ Args:
76
+ prompts: A list of prompts to generate completions for.
77
+ sampling_params: The sampling parameters for text generation. If
78
+ None, we use the default sampling parameters.
79
+ prompt_token_ids: A list of token IDs for the prompts. If None, we
80
+ use the tokenizer to convert the prompts to token IDs.
81
+ use_tqdm: Whether to use tqdm to display the progress bar.
82
+
83
+ Returns:
84
+ A list of `RequestOutput` objects containing the generated
85
+ completions in the same order as the input prompts.
86
+ """
87
+ if prompts is None and prompt_token_ids is None:
88
+ raise ValueError("Either prompts or prompt_token_ids must be "
89
+ "provided.")
90
if isinstance(prompts, str):
91
+ # Convert a single prompt to a list.
92
prompts = [prompts]
93
+ if prompts is not None and prompt_token_ids is not None:
94
+ if len(prompts) != len(prompt_token_ids):
95
+ raise ValueError("The lengths of prompts and prompt_token_ids "
96
+ "must be the same.")
97
if sampling_params is None:
98
# Use default sampling params.
99
sampling_params = SamplingParams()
100
+
101
# Add requests to the server.
53
- for i in range(len(prompts)):
54
- prompt = prompts[i]
102
+ if prompts is not None:
103
+ num_requests = len(prompts)
104
+ else:
105
+ num_requests = len(prompt_token_ids)
106
+ for i in range(num_requests):
107
+ prompt = prompts[i] if prompts is not None else None
108
if prompt_token_ids is None:
109
token_ids = None
110
else:
@@ -61,7 +114,7 @@ class LLM:
114
115
def _add_request(
116
self,
64
- prompt: str,
117
+ prompt: Optional[str],
118
sampling_params: SamplingParams,
119
prompt_token_ids: Optional[List[int]],
120
) -> None:
cacheflow/server/llm_server.py
+2
-1
@@ -126,7 +126,7 @@ class LLMServer:
126
def add_request(
127
self,
128
request_id: str,
129
- prompt: str,
129
+ prompt: Optional[str],
130
sampling_params: SamplingParams,
131
prompt_token_ids: Optional[List[int]] = None,
132
arrival_time: Optional[float] = None,
@@ -134,6 +134,7 @@ class LLMServer:
134
if arrival_time is None:
135
arrival_time = time.time()
136
if prompt_token_ids is None:
137
+ assert prompt is not None
138
prompt_token_ids = self.tokenizer.encode(prompt)
139
140
# Create the sequences.