@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8274ca23ac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add docstrings for LLM (#137)

Woosuk Kwon committed Jun 4, 2023 at 12:52 UTC 8274ca23ac9f2ea0ccf758d1883794643aecb2e0
4 files changed +66 -10
benchmarks/benchmark_latency.py
+2 -2
@@ -30,7 +30,6 @@ def main(args: argparse.Namespace):
30 max_tokens=args.output_len,
31 )
32 print(sampling_params)
33 - dummy_prompts = [""] * args.batch_size
33 dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
34
35 def run_to_completion(profile: bool = False):
@@ -38,7 +37,8 @@ def main(args: argparse.Namespace):
37 torch.cuda.cudart().cudaProfilerStart()
38 start_time = time.time()
39
41 - llm.generate(dummy_prompts, sampling_params, dummy_prompt_token_ids,
40 + llm.generate(prompt_token_ids=dummy_prompt_token_ids,
41 + sampling_params=sampling_params,
42 use_tqdm=False)
43
44 end_time = time.time()
benchmarks/benchmark_throughput.py
+5 -3
@@ -72,9 +72,9 @@ def main(args: argparse.Namespace):
72 )
73 # FIXME(woosuk): Do not use internal method.
74 llm._add_request(
75 - prompt="",
76 - sampling_params=sampling_params,
75 + prompt=None,
76 prompt_token_ids=prompt_token_ids,
77 + sampling_params=sampling_params,
78 )
79
80 start = time.time()
@@ -85,7 +85,9 @@ def main(args: argparse.Namespace):
85 len(prompt_token_ids) + output_len
86 for prompt_token_ids, output_len in requests
87 )
88 - print(f"Throughput: {total_num_tokens / (end - start):.2f} tokens/s")
88 + elapsed_time = end - start
89 + print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
90 + f"{total_num_tokens / elapsed_time:.2f} tokens/s")
91
92
93 if __name__ == "__main__":
cacheflow/entrypoints/llm.py
+57 -4
@@ -11,6 +11,28 @@ from cacheflow.utils import Counter
11
12
13 class LLM:
14 + """An LLM for generating texts from given prompts and sampling parameters.
15 +
16 + This class includes a tokenizer, a language model (possibly distributed
17 + across multiple GPUs), and GPU memory space allocated for intermediate
18 + states (aka KV cache). Given a batch of prompts and sampling parameters,
19 + this class generates texts from the model, using an intelligent batching
20 + mechanism and efficient memory management.
21 +
22 + NOTE: This class is intended to be used for offline inference. For online
23 + serving, use the `AsyncLLMServer` class instead.
24 + NOTE: For the comprehensive list of arguments, see `ServerArgs`.
25 +
26 + Args:
27 + model: The name or path of a HuggingFace Transformers model.
28 + tensor_parallel_size: The number of GPUs to use for distributed
29 + execution with tensor parallelism.
30 + dtype: The data type for the model weights and activations. Currently,
31 + we support `float16` and `bfloat16`. If `default`, we use the
32 + `torch_dtype` attribute of the model config. If the `torch_dtype`
33 + is `float32`, we use `float16` instead.
34 + seed: The seed to initialize the random number generator for sampling.
35 + """
36
37 def __init__(
38 self,
@@ -39,19 +61,50 @@ class LLM:
61
62 def generate(
63 self,
42 - prompts: Union[str, List[str]],
64 + prompts: Optional[Union[str, List[str]]] = None,
65 sampling_params: Optional[SamplingParams] = None,
66 prompt_token_ids: Optional[List[List[int]]] = None,
67 use_tqdm: bool = True,
68 ) -> List[RequestOutput]:
69 + """Generates the completions for the input prompts.
70 +
71 + NOTE: This class automatically batches the given prompts, considering
72 + the memory constraint. For the best performance, put all of your prompts
73 + into a single list and pass it to this method.
74 +
75 + Args:
76 + prompts: A list of prompts to generate completions for.
77 + sampling_params: The sampling parameters for text generation. If
78 + None, we use the default sampling parameters.
79 + prompt_token_ids: A list of token IDs for the prompts. If None, we
80 + use the tokenizer to convert the prompts to token IDs.
81 + use_tqdm: Whether to use tqdm to display the progress bar.
82 +
83 + Returns:
84 + A list of `RequestOutput` objects containing the generated
85 + completions in the same order as the input prompts.
86 + """
87 + if prompts is None and prompt_token_ids is None:
88 + raise ValueError("Either prompts or prompt_token_ids must be "
89 + "provided.")
90 if isinstance(prompts, str):
91 + # Convert a single prompt to a list.
92 prompts = [prompts]
93 + if prompts is not None and prompt_token_ids is not None:
94 + if len(prompts) != len(prompt_token_ids):
95 + raise ValueError("The lengths of prompts and prompt_token_ids "
96 + "must be the same.")
97 if sampling_params is None:
98 # Use default sampling params.
99 sampling_params = SamplingParams()
100 +
101 # Add requests to the server.
53 - for i in range(len(prompts)):
54 - prompt = prompts[i]
102 + if prompts is not None:
103 + num_requests = len(prompts)
104 + else:
105 + num_requests = len(prompt_token_ids)
106 + for i in range(num_requests):
107 + prompt = prompts[i] if prompts is not None else None
108 if prompt_token_ids is None:
109 token_ids = None
110 else:
@@ -61,7 +114,7 @@ class LLM:
114
115 def _add_request(
116 self,
64 - prompt: str,
117 + prompt: Optional[str],
118 sampling_params: SamplingParams,
119 prompt_token_ids: Optional[List[int]],
120 ) -> None:
cacheflow/server/llm_server.py
+2 -1
@@ -126,7 +126,7 @@ class LLMServer:
126 def add_request(
127 self,
128 request_id: str,
129 - prompt: str,
129 + prompt: Optional[str],
130 sampling_params: SamplingParams,
131 prompt_token_ids: Optional[List[int]] = None,
132 arrival_time: Optional[float] = None,
@@ -134,6 +134,7 @@ class LLMServer:
134 if arrival_time is None:
135 arrival_time = time.time()
136 if prompt_token_ids is None:
137 + assert prompt is not None
138 prompt_token_ids = self.tokenizer.encode(prompt)
139
140 # Create the sequences.