@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6fa1be3a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Quality] Add code formatter and linter (#326)

Zhuohan Li committed Jul 3, 2023 at 11:31 UTC d6fa1be3a8ef71fa16f74afdc5d07d27cbf725b1
47 files changed +1549 -619
.pylintrc new
+434
@@ -0,0 +1,434 @@
1 +# This Pylint rcfile contains a best-effort configuration to uphold the
2 +# best-practices and style described in the Google Python style guide:
3 +# https://google.github.io/styleguide/pyguide.html
4 +#
5 +# Its canonical open-source location is:
6 +# https://google.github.io/styleguide/pylintrc
7 +
8 +[MASTER]
9 +
10 +# Files or directories to be skipped. They should be base names, not paths.
11 +ignore=docs,parallel_utils
12 +
13 +# Files or directories matching the regex patterns are skipped. The regex
14 +# matches against base names, not paths.
15 +ignore-patterns=
16 +
17 +# Pickle collected data for later comparisons.
18 +persistent=no
19 +
20 +# List of plugins (as comma separated values of python modules names) to load,
21 +# usually to register additional checkers.
22 +load-plugins=
23 +
24 +# Use multiple processes to speed up Pylint.
25 +jobs=4
26 +
27 +# Allow loading of arbitrary C extensions. Extensions are imported into the
28 +# active Python interpreter and may run arbitrary code.
29 +unsafe-load-any-extension=no
30 +
31 +
32 +[MESSAGES CONTROL]
33 +
34 +# Only show warnings with the listed confidence levels. Leave empty to show
35 +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
36 +confidence=
37 +
38 +# Enable the message, report, category or checker with the given id(s). You can
39 +# either give multiple identifier separated by comma (,) or put this option
40 +# multiple time (only on the command line, not in the configuration file where
41 +# it should appear only once). See also the "--disable" option for examples.
42 +#enable=
43 +
44 +# Disable the message, report, category or checker with the given id(s). You
45 +# can either give multiple identifiers separated by comma (,) or put this
46 +# option multiple times (only on the command line, not in the configuration
47 +# file where it should appear only once).You can also use "--disable=all" to
48 +# disable everything first and then reenable specific checks. For example, if
49 +# you want to run only the similarities checker, you can use "--disable=all
50 +# --enable=similarities". If you want to run only the classes checker, but have
51 +# no Warning level messages displayed, use"--disable=all --enable=classes
52 +# --disable=W"
53 +disable=abstract-method,
54 + apply-builtin,
55 + arguments-differ,
56 + attribute-defined-outside-init,
57 + backtick,
58 + bad-option-value,
59 + basestring-builtin,
60 + buffer-builtin,
61 + c-extension-no-member,
62 + consider-using-enumerate,
63 + cmp-builtin,
64 + cmp-method,
65 + coerce-builtin,
66 + coerce-method,
67 + delslice-method,
68 + div-method,
69 + duplicate-code,
70 + eq-without-hash,
71 + execfile-builtin,
72 + file-builtin,
73 + filter-builtin-not-iterating,
74 + fixme,
75 + getslice-method,
76 + global-statement,
77 + hex-method,
78 + idiv-method,
79 + implicit-str-concat-in-sequence,
80 + import-error,
81 + import-self,
82 + import-star-module-level,
83 + inconsistent-return-statements,
84 + input-builtin,
85 + intern-builtin,
86 + invalid-str-codec,
87 + locally-disabled,
88 + logging-fstring-interpolation, # added by vLLM
89 + logging-not-lazy, # added by vLLM
90 + long-builtin,
91 + long-suffix,
92 + map-builtin-not-iterating,
93 + misplaced-comparison-constant,
94 + missing-class-docstring, # TODO (vLLM): enable
95 + missing-function-docstring,
96 + missing-module-docstring, # TODO (vLLM): enable
97 + metaclass-assignment,
98 + next-method-called,
99 + next-method-defined,
100 + no-absolute-import,
101 + no-else-break,
102 + no-else-continue,
103 + no-else-raise,
104 + no-else-return,
105 + no-init, # added
106 + no-member,
107 + no-name-in-module,
108 + no-self-use,
109 + nonzero-method,
110 + oct-method,
111 + old-division,
112 + old-ne-operator,
113 + old-octal-literal,
114 + old-raise-syntax,
115 + parameter-unpacking,
116 + print-statement,
117 + raising-string,
118 + range-builtin-not-iterating,
119 + raw_input-builtin,
120 + rdiv-method,
121 + reduce-builtin,
122 + relative-import,
123 + reload-builtin,
124 + round-builtin,
125 + setslice-method,
126 + signature-differs,
127 + standarderror-builtin,
128 + suppressed-message,
129 + sys-max-int,
130 + too-few-public-methods,
131 + too-many-ancestors,
132 + too-many-arguments,
133 + too-many-boolean-expressions,
134 + too-many-branches,
135 + too-many-instance-attributes,
136 + too-many-locals,
137 + too-many-nested-blocks,
138 + too-many-public-methods,
139 + too-many-return-statements,
140 + too-many-statements,
141 + trailing-newlines,
142 + unichr-builtin,
143 + unicode-builtin,
144 + unnecessary-pass,
145 + unpacking-in-except,
146 + unspecified-encoding,
147 + useless-else-on-loop,
148 + useless-object-inheritance,
149 + useless-suppression,
150 + using-cmp-argument,
151 + wrong-import-order,
152 + xrange-builtin,
153 + zip-builtin-not-iterating,
154 +
155 +
156 +[REPORTS]
157 +
158 +# Set the output format. Available formats are text, parseable, colorized, msvs
159 +# (visual studio) and html. You can also give a reporter class, eg
160 +# mypackage.mymodule.MyReporterClass.
161 +output-format=text
162 +
163 +# Tells whether to display a full report or only the messages
164 +reports=no
165 +
166 +# Python expression which should return a note less than 10 (10 is the highest
167 +# note). You have access to the variables errors warning, statement which
168 +# respectively contain the number of errors / warnings messages and the total
169 +# number of statements analyzed. This is used by the global evaluation report
170 +# (RP0004).
171 +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
172 +
173 +# Template used to display messages. This is a python new-style format string
174 +# used to format the message information. See doc for all details
175 +#msg-template=
176 +
177 +
178 +[BASIC]
179 +
180 +# Good variable names which should always be accepted, separated by a comma
181 +good-names=main,_
182 +
183 +# Bad variable names which should always be refused, separated by a comma
184 +bad-names=
185 +
186 +# Colon-delimited sets of names that determine each other's naming style when
187 +# the name regexes allow several styles.
188 +name-group=
189 +
190 +# Include a hint for the correct naming format with invalid-name
191 +include-naming-hint=no
192 +
193 +# List of decorators that produce properties, such as abc.abstractproperty. Add
194 +# to this list to register other decorators that produce valid properties.
195 +property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl
196 +
197 +# Regular expression matching correct function names
198 +function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
199 +
200 +# Regular expression matching correct variable names
201 +variable-rgx=^[a-z][a-z0-9_]*$
202 +
203 +# Regular expression matching correct constant names
204 +const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
205 +
206 +# Regular expression matching correct attribute names
207 +attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
208 +
209 +# Regular expression matching correct argument names
210 +argument-rgx=^[a-z][a-z0-9_]*$
211 +
212 +# Regular expression matching correct class attribute names
213 +class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
214 +
215 +# Regular expression matching correct inline iteration names
216 +inlinevar-rgx=^[a-z][a-z0-9_]*$
217 +
218 +# Regular expression matching correct class names
219 +class-rgx=^_?[A-Z][a-zA-Z0-9]*$
220 +
221 +# Regular expression matching correct module names
222 +module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$
223 +
224 +# Regular expression matching correct method names
225 +method-rgx=(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
226 +
227 +# Regular expression which should only match function or class names that do
228 +# not require a docstring.
229 +no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$
230 +
231 +# Minimum line length for functions/classes that require docstrings, shorter
232 +# ones are exempt.
233 +docstring-min-length=10
234 +
235 +
236 +[TYPECHECK]
237 +
238 +# List of decorators that produce context managers, such as
239 +# contextlib.contextmanager. Add to this list to register other decorators that
240 +# produce valid context managers.
241 +contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager
242 +
243 +# Tells whether missing members accessed in mixin class should be ignored. A
244 +# mixin class is detected if its name ends with "mixin" (case insensitive).
245 +ignore-mixin-members=yes
246 +
247 +# List of module names for which member attributes should not be checked
248 +# (useful for modules/projects where namespaces are manipulated during runtime
249 +# and thus existing member attributes cannot be deduced by static analysis. It
250 +# supports qualified module names, as well as Unix pattern matching.
251 +ignored-modules=
252 +
253 +# List of class names for which member attributes should not be checked (useful
254 +# for classes with dynamically set attributes). This supports the use of
255 +# qualified names.
256 +ignored-classes=optparse.Values,thread._local,_thread._local
257 +
258 +# List of members which are set dynamically and missed by pylint inference
259 +# system, and so shouldn't trigger E1101 when accessed. Python regular
260 +# expressions are accepted.
261 +generated-members=
262 +
263 +
264 +[FORMAT]
265 +
266 +# Maximum number of characters on a single line.
267 +max-line-length=80
268 +
269 +# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt
270 +# lines made too long by directives to pytype.
271 +
272 +# Regexp for a line that is allowed to be longer than the limit.
273 +ignore-long-lines=(?x)(
274 + ^\s*(\#\ )?<?https?://\S+>?$|
275 + ^\s*(from\s+\S+\s+)?import\s+.+$)
276 +
277 +# Allow the body of an if to be on the same line as the test if there is no
278 +# else.
279 +single-line-if-stmt=yes
280 +
281 +# Maximum number of lines in a module
282 +max-module-lines=99999
283 +
284 +# String used as indentation unit. The internal Google style guide mandates 2
285 +# spaces. Google's externaly-published style guide says 4, consistent with
286 +# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google
287 +# projects (like TensorFlow).
288 +indent-string=' '
289 +
290 +# Number of spaces of indent required inside a hanging or continued line.
291 +indent-after-paren=4
292 +
293 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
294 +expected-line-ending-format=
295 +
296 +
297 +[MISCELLANEOUS]
298 +
299 +# List of note tags to take in consideration, separated by a comma.
300 +notes=TODO
301 +
302 +
303 +[STRING]
304 +
305 +# This flag controls whether inconsistent-quotes generates a warning when the
306 +# character used as a quote delimiter is used inconsistently within a module.
307 +check-quote-consistency=yes
308 +
309 +
310 +[VARIABLES]
311 +
312 +# Tells whether we should check for unused import in __init__ files.
313 +init-import=no
314 +
315 +# A regular expression matching the name of dummy variables (i.e. expectedly
316 +# not used).
317 +dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_)
318 +
319 +# List of additional names supposed to be defined in builtins. Remember that
320 +# you should avoid to define new builtins when possible.
321 +additional-builtins=
322 +
323 +# List of strings which can identify a callback function by name. A callback
324 +# name must start or end with one of those strings.
325 +callbacks=cb_,_cb
326 +
327 +# List of qualified module names which can have objects that can redefine
328 +# builtins.
329 +redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools
330 +
331 +
332 +[LOGGING]
333 +
334 +# Logging modules to check that the string format arguments are in logging
335 +# function parameter format
336 +logging-modules=logging,absl.logging,tensorflow.io.logging
337 +
338 +
339 +[SIMILARITIES]
340 +
341 +# Minimum lines number of a similarity.
342 +min-similarity-lines=4
343 +
344 +# Ignore comments when computing similarities.
345 +ignore-comments=yes
346 +
347 +# Ignore docstrings when computing similarities.
348 +ignore-docstrings=yes
349 +
350 +# Ignore imports when computing similarities.
351 +ignore-imports=no
352 +
353 +
354 +[SPELLING]
355 +
356 +# Spelling dictionary name. Available dictionaries: none. To make it working
357 +# install python-enchant package.
358 +spelling-dict=
359 +
360 +# List of comma separated words that should not be checked.
361 +spelling-ignore-words=
362 +
363 +# A path to a file that contains private dictionary; one word per line.
364 +spelling-private-dict-file=
365 +
366 +# Tells whether to store unknown words to indicated private dictionary in
367 +# --spelling-private-dict-file option instead of raising a message.
368 +spelling-store-unknown-words=no
369 +
370 +
371 +[IMPORTS]
372 +
373 +# Deprecated modules which should not be used, separated by a comma
374 +deprecated-modules=regsub,
375 + TERMIOS,
376 + Bastion,
377 + rexec,
378 + sets
379 +
380 +# Create a graph of every (i.e. internal and external) dependencies in the
381 +# given file (report RP0402 must not be disabled)
382 +import-graph=
383 +
384 +# Create a graph of external dependencies in the given file (report RP0402 must
385 +# not be disabled)
386 +ext-import-graph=
387 +
388 +# Create a graph of internal dependencies in the given file (report RP0402 must
389 +# not be disabled)
390 +int-import-graph=
391 +
392 +# Force import order to recognize a module as part of the standard
393 +# compatibility libraries.
394 +known-standard-library=
395 +
396 +# Force import order to recognize a module as part of a third party library.
397 +known-third-party=enchant, absl
398 +
399 +# Analyse import fallback blocks. This can be used to support both Python 2 and
400 +# 3 compatible code, which means that the block might have code that exists
401 +# only in one or another interpreter, leading to false positives when analysed.
402 +analyse-fallback-blocks=no
403 +
404 +
405 +[CLASSES]
406 +
407 +# List of method names used to declare (i.e. assign) instance attributes.
408 +defining-attr-methods=__init__,
409 + __new__,
410 + setUp
411 +
412 +# List of member names, which should be excluded from the protected access
413 +# warning.
414 +exclude-protected=_asdict,
415 + _fields,
416 + _replace,
417 + _source,
418 + _make
419 +
420 +# List of valid names for the first argument in a class method.
421 +valid-classmethod-first-arg=cls,
422 + class_
423 +
424 +# List of valid names for the first argument in a metaclass class method.
425 +valid-metaclass-classmethod-first-arg=mcs
426 +
427 +
428 +[EXCEPTIONS]
429 +
430 +# Exceptions that will emit a warning when being caught. Defaults to
431 +# "Exception"
432 +overgeneral-exceptions=StandardError,
433 + Exception,
434 + BaseException
CONTRIBUTING.md
+4 -1
@@ -49,12 +49,15 @@ If not, please file a new issue, providing as much relevant information as possi
49
50 In general, we adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html) and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
51
52 +We include a formatting script [`format.sh`](./format.sh) to format the code.
53 +
54 ### Pull Requests
55
56 When submitting a pull request:
57
58 1. Make sure your code has been rebased on top of the latest commit on the main branch.
57 -2. Include a detailed description of the changes in the pull request.
59 +2. Ensure code is properly formatted by running [`format.sh`](./format.sh).
60 +3. Include a detailed description of the changes in the pull request.
61 Explain why you made the changes you did.
62 If your pull request fixes an open issue, please include a reference to it in the description.
63
examples/api_client.py
+5 -2
@@ -14,7 +14,9 @@ def clear_line(n: int = 1) -> None:
14 print(LINE_UP, end=LINE_CLEAR, flush=True)
15
16
17 -def post_http_request(prompt: str, api_url: str, n: int = 1,
17 +def post_http_request(prompt: str,
18 + api_url: str,
19 + n: int = 1,
20 stream: bool = False) -> requests.Response:
21 headers = {"User-Agent": "Test Client"}
22 pload = {
@@ -30,7 +32,8 @@ def post_http_request(prompt: str, api_url: str, n: int = 1,
32
33
34 def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:
33 - for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False,
35 + for chunk in response.iter_lines(chunk_size=8192,
36 + decode_unicode=False,
37 delimiter=b"\0"):
38 if chunk:
39 data = json.loads(chunk.decode("utf-8"))
examples/gradio_webserver.py
+16 -9
@@ -12,9 +12,14 @@ def http_bot(prompt):
12 "stream": True,
13 "max_tokens": 128,
14 }
15 - response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
16 -
17 - for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
15 + response = requests.post(args.model_url,
16 + headers=headers,
17 + json=pload,
18 + stream=True)
19 +
20 + for chunk in response.iter_lines(chunk_size=8192,
21 + decode_unicode=False,
22 + delimiter=b"\0"):
23 if chunk:
24 data = json.loads(chunk.decode("utf-8"))
25 output = data["text"][0]
@@ -23,11 +28,11 @@ def http_bot(prompt):
28
29 def build_demo():
30 with gr.Blocks() as demo:
26 - gr.Markdown(
27 - "# vLLM text completion demo\n"
28 - )
29 - inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")
30 - outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
31 + gr.Markdown("# vLLM text completion demo\n")
32 + inputbox = gr.Textbox(label="Input",
33 + placeholder="Enter text and press ENTER")
34 + outputbox = gr.Textbox(label="Output",
35 + placeholder="Generated result from the model")
36 inputbox.submit(http_bot, [inputbox], [outputbox])
37 return demo
38
@@ -36,7 +41,9 @@ if __name__ == "__main__":
41 parser = argparse.ArgumentParser()
42 parser.add_argument("--host", type=str, default="localhost")
43 parser.add_argument("--port", type=int, default=8001)
39 - parser.add_argument("--model-url", type=str, default="http://localhost:8000/generate")
44 + parser.add_argument("--model-url",
45 + type=str,
46 + default="http://localhost:8000/generate")
47 args = parser.parse_args()
48
49 demo = build_demo()
examples/llm_engine_example.py
+7 -2
@@ -14,9 +14,14 @@ def main(args: argparse.Namespace):
14 ("To be or not to be,",
15 SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
16 ("What is the meaning of life?",
17 - SamplingParams(n=2, best_of=5, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
17 + SamplingParams(n=2,
18 + best_of=5,
19 + temperature=0.8,
20 + top_p=0.95,
21 + frequency_penalty=0.1)),
22 ("It is only with the heart that one can see rightly",
19 - SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
23 + SamplingParams(n=3, best_of=3, use_beam_search=True,
24 + temperature=0.0)),
25 ]
26
27 # Run the engine by calling `engine.step()` manually.
examples/offline_inference.py
-1
@@ -1,6 +1,5 @@
1 from vllm import LLM, SamplingParams
2
3 -
3 # Sample prompts.
4 prompts = [
5 "Hello, my name is",
examples/openai_client.py
+7 -2
@@ -12,8 +12,13 @@ print("Models:", models)
12 # Test completion API
13 stream = True
14 completion = openai.Completion.create(
15 - model=model, prompt="A robot may not injure a human being", echo=False, n=2,
16 - best_of=3, stream=stream, logprobs=3)
15 + model=model,
16 + prompt="A robot may not injure a human being",
17 + echo=False,
18 + n=2,
19 + best_of=3,
20 + stream=stream,
21 + logprobs=3)
22
23 # print the completion
24 if stream:
format.sh new
+108
@@ -0,0 +1,108 @@
1 +#!/usr/bin/env bash
2 +# YAPF formatter, adapted from ray and skypilot.
3 +#
4 +# Usage:
5 +# # Do work and commit your work.
6 +
7 +# # Format files that differ from origin/main.
8 +# bash format.sh
9 +
10 +# # Commit changed files with message 'Run yapf and pylint'
11 +#
12 +#
13 +# YAPF + Clang formatter (if installed). This script formats all changed files from the last mergebase.
14 +# You are encouraged to run this locally before pushing changes for review.
15 +
16 +# Cause the script to exit if a single command fails
17 +set -eo pipefail
18 +
19 +# this stops git rev-parse from failing if we run this from the .git directory
20 +builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
21 +ROOT="$(git rev-parse --show-toplevel)"
22 +builtin cd "$ROOT" || exit 1
23 +
24 +YAPF_VERSION=$(yapf --version | awk '{print $2}')
25 +PYLINT_VERSION=$(pylint --version | head -n 1 | awk '{print $2}')
26 +MYPY_VERSION=$(mypy --version | awk '{print $2}')
27 +
28 +# # params: tool name, tool version, required version
29 +tool_version_check() {
30 + if [[ $2 != $3 ]]; then
31 + echo "Wrong $1 version installed: $3 is required, not $2."
32 + exit 1
33 + fi
34 +}
35 +
36 +tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)"
37 +tool_version_check "pylint" $PYLINT_VERSION "$(grep "pylint==" requirements-dev.txt | cut -d'=' -f3)"
38 +tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
39 +
40 +YAPF_FLAGS=(
41 + '--recursive'
42 + '--parallel'
43 +)
44 +
45 +YAPF_EXCLUDES=(
46 + '--exclude' 'build/**'
47 + '--exclude' 'vllm/model_executor/parallel_utils/**'
48 +)
49 +
50 +# Format specified files
51 +format() {
52 + yapf --in-place "${YAPF_FLAGS[@]}" "$@"
53 +}
54 +
55 +# Format files that differ from main branch. Ignores dirs that are not slated
56 +# for autoformat yet.
57 +format_changed() {
58 + # The `if` guard ensures that the list of filenames is not empty, which
59 + # could cause yapf to receive 0 positional arguments, making it hang
60 + # waiting for STDIN.
61 + #
62 + # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
63 + # exist on both branches.
64 + MERGEBASE="$(git merge-base origin/main HEAD)"
65 +
66 + if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
67 + git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs -P 5 \
68 + yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
69 + fi
70 +
71 +}
72 +
73 +# Format all files
74 +format_all() {
75 + yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" vllm
76 +}
77 +
78 +## This flag formats individual files. --files *must* be the first command line
79 +## arg to use this option.
80 +if [[ "$1" == '--files' ]]; then
81 + format "${@:2}"
82 + # If `--all` is passed, then any further arguments are ignored and the
83 + # entire python directory is formatted.
84 +elif [[ "$1" == '--all' ]]; then
85 + format_all
86 +else
87 + # Format only the files that changed in last commit.
88 + format_changed
89 +fi
90 +echo 'vLLM yapf: Done'
91 +
92 +# Run mypy
93 +# TODO(zhuohan): Enable mypy
94 +# echo 'vLLM mypy:'
95 +# mypy
96 +
97 +# Run Pylint
98 +echo 'vLLM Pylint:'
99 +pylint vllm
100 +
101 +if ! git diff --quiet &>/dev/null; then
102 + echo 'Reformatted files. Please review and stage the changes.'
103 + echo 'Changes not staged for commit:'
104 + echo
105 + git --no-pager diff --name-only
106 +
107 + exit 1
108 +fi
requirements-dev.txt
+11 -1
@@ -1,2 +1,12 @@
1 -mypy
1 +# formatting
2 +yapf==0.32.0
3 +pylint==2.8.2
4 +
5 +# type checking
6 +mypy==0.991
7 +types-PyYAML
8 +types-requests
9 +types-setuptools
10 +
11 +# testing
12 pytest
tests/kernels/test_attention.py
+33 -20
@@ -60,7 +60,7 @@ def ref_single_query_cached_kv_attention(
60 keys = torch.stack(keys, dim=0)
61 values = torch.stack(values, dim=0)
62
63 - scale = 1.0 / (head_size ** 0.5)
63 + scale = 1.0 / (head_size**0.5)
64 out = ref_masked_attention(q, keys, values, scale)
65 out = out.view(num_heads, head_size)
66 output[i].copy_(out, non_blocking=True)
@@ -74,7 +74,7 @@ def ref_multi_query_kv_attention(
74 dtype: torch.dtype,
75 ) -> torch.Tensor:
76 head_size = query.shape[-1]
77 - scale = 1.0 / (head_size ** 0.5)
77 + scale = 1.0 / (head_size**0.5)
78
79 num_seqs = len(cu_seq_lens) - 1
80 ref_outputs = []
@@ -84,8 +84,8 @@ def ref_multi_query_kv_attention(
84 seq_len = end_idx - start_idx
85
86 # Create attention mask.
87 - attn_mask = torch.triu(
88 - torch.ones(seq_len, seq_len, dtype=dtype), diagonal=1)
87 + attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
88 + diagonal=1)
89 attn_mask = attn_mask * torch.finfo(dtype).min
90 attn_mask = attn_mask.to(dtype=dtype, device='cuda')
91
@@ -113,7 +113,7 @@ def ref_multi_query_cached_kv_attention(
113 num_heads = value_cache.shape[1]
114 head_size = value_cache.shape[2]
115 block_size = value_cache.shape[3]
116 - scale = 1.0 / (head_size ** 0.5)
116 + scale = 1.0 / (head_size**0.5)
117
118 num_queries = len(cu_query_lens) - 1
119 ref_outputs = []
@@ -125,8 +125,8 @@ def ref_multi_query_cached_kv_attention(
125 block_table = block_tables[i]
126
127 # Create attention mask
128 - attn_mask = torch.triu(
129 - torch.ones(query_len, context_len), diagonal=context_len - query_len + 1) * -1e5
128 + attn_mask = torch.triu(torch.ones(query_len, context_len),
129 + diagonal=context_len - query_len + 1) * -1e5
130 attn_mask = attn_mask.to(dtype=dtype, device='cuda')
131
132 keys = []
@@ -165,22 +165,28 @@ def run_single_query_cached_kv_attention(
165 num_blocks: int,
166 dtype: torch.dtype,
167 ) -> None:
168 - qkv = torch.empty(
169 - num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
168 + qkv = torch.empty(num_tokens,
169 + 3,
170 + num_heads,
171 + head_size,
172 + dtype=dtype,
173 + device='cuda')
174 qkv.uniform_(-1e-3, 1e-3)
175 query, _, _ = qkv.unbind(dim=1)
176
177 x = 16 // torch.tensor([], dtype=dtype).element_size()
178 key_block_shape = (num_heads, head_size // x, block_size, x)
175 - key_cache = torch.empty(
176 - size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
179 + key_cache = torch.empty(size=(num_blocks, *key_block_shape),
180 + dtype=dtype,
181 + device='cuda')
182 key_cache.uniform_(-1e-3, 1e-3)
183 value_block_shape = (num_heads, head_size, block_size)
179 - value_cache = torch.empty(
180 - size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
184 + value_cache = torch.empty(size=(num_blocks, *value_block_shape),
185 + dtype=dtype,
186 + device='cuda')
187 value_cache.uniform_(-1e-3, 1e-3)
188
183 - context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
189 + context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
190 max_context_len = max(context_lens)
191 context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
192
@@ -194,9 +200,12 @@ def run_single_query_cached_kv_attention(
200 block_tables.append(block_table)
201 block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
202
197 - scale = float(1.0 / (head_size ** 0.5))
198 - output = torch.empty(
199 - num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
203 + scale = float(1.0 / (head_size**0.5))
204 + output = torch.empty(num_tokens,
205 + num_heads,
206 + head_size,
207 + dtype=dtype,
208 + device='cuda')
209 attention_ops.single_query_cached_kv_attention(
210 output,
211 query,
@@ -235,9 +244,13 @@ def run_multi_query_kv_attention(
244 seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
245 num_tokens = sum(seq_lens)
246
238 - scale = float(1.0 / (head_size ** 0.5))
239 - qkv = torch.empty(
240 - num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
247 + scale = float(1.0 / (head_size**0.5))
248 + qkv = torch.empty(num_tokens,
249 + 3,
250 + num_heads,
251 + head_size,
252 + dtype=dtype,
253 + device='cuda')
254 qkv.uniform_(-1e-3, 1e-3)
255 query, key, value = qkv.unbind(dim=1)
256
tests/kernels/test_cache.py
+61 -29
@@ -26,8 +26,9 @@ def run_copy_blocks(
26 key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
27 key_caches = []
28 for _ in range(num_layers):
29 - key_cache = torch.randn(
30 - size=key_cache_shape, dtype=dtype, device='cuda')
29 + key_cache = torch.randn(size=key_cache_shape,
30 + dtype=dtype,
31 + device='cuda')
32 key_caches.append(key_cache)
33 cloned_key_caches = []
34 for key_cache in key_caches:
@@ -36,8 +37,9 @@ def run_copy_blocks(
37 value_cache_shape = (num_blocks, num_heads, head_size, block_size)
38 value_caches = []
39 for _ in range(num_layers):
39 - value_cache = torch.randn(
40 - size=value_cache_shape, dtype=dtype, device='cuda')
40 + value_cache = torch.randn(size=value_cache_shape,
41 + dtype=dtype,
42 + device='cuda')
43 value_caches.append(value_cache)
44 cloned_value_caches = []
45 for value_cache in value_caches:
@@ -49,15 +51,18 @@ def run_copy_blocks(
51 # Reference implementation.
52 for src, dsts in block_mapping.items():
53 for dst in dsts:
52 - for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
54 + for key_cache, cloned_key_cache in zip(key_caches,
55 + cloned_key_caches):
56 cloned_key_cache[dst] = cloned_key_cache[src]
54 - for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
57 + for value_cache, cloned_value_cache in zip(value_caches,
58 + cloned_value_caches):
59 cloned_value_cache[dst] = cloned_value_cache[src]
60
61 # Compare the results.
62 for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
63 assert torch.allclose(key_cache, cloned_key_cache)
60 - for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
64 + for value_cache, cloned_value_cache in zip(value_caches,
65 + cloned_value_caches):
66 assert torch.allclose(value_cache, cloned_value_cache)
67
68
@@ -74,8 +79,12 @@ def run_reshape_and_cache(
79 slot_mapping = random.sample(range(num_slots), num_tokens)
80 slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
81
77 - qkv = torch.randn(
78 - num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
82 + qkv = torch.randn(num_tokens,
83 + 3,
84 + num_heads,
85 + head_size,
86 + dtype=dtype,
87 + device='cuda')
88 _, key, value = qkv.unbind(dim=1)
89
90 x = 16 // torch.tensor([], dtype=dtype).element_size()
@@ -84,15 +93,19 @@ def run_reshape_and_cache(
93 cloned_key_cache = key_cache.clone()
94
95 value_cache_shape = (num_blocks, num_heads, head_size, block_size)
87 - value_cache = torch.randn(
88 - size=value_cache_shape, dtype=dtype, device='cuda')
96 + value_cache = torch.randn(size=value_cache_shape,
97 + dtype=dtype,
98 + device='cuda')
99 cloned_value_cache = value_cache.clone()
100
91 - cache_ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
101 + cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
102 + slot_mapping)
103
104 for i in range(num_tokens):
105 reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
95 - block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
106 + block_idx = torch.div(slot_mapping[i],
107 + block_size,
108 + rounding_mode='floor')
109 block_offset = slot_mapping[i] % block_size
110 cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
111 cloned_value_cache[block_idx, :, :, block_offset] = value[i]
@@ -114,8 +127,12 @@ def run_gather_cached_kv(
127 slot_mapping = random.sample(range(num_slots), num_tokens)
128 slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
129
117 - qkv = torch.randn(
118 - num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
130 + qkv = torch.randn(num_tokens,
131 + 3,
132 + num_heads,
133 + head_size,
134 + dtype=dtype,
135 + device='cuda')
136 _, key, value = qkv.unbind(dim=1)
137
138 qkv_clone = qkv.clone()
@@ -126,15 +143,20 @@ def run_gather_cached_kv(
143 key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
144
145 value_cache_shape = (num_blocks, num_heads, head_size, block_size)
129 - value_cache = torch.randn(
130 - size=value_cache_shape, dtype=dtype, device='cuda')
146 + value_cache = torch.randn(size=value_cache_shape,
147 + dtype=dtype,
148 + device='cuda')
149
132 - cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
150 + cache_ops.gather_cached_kv(key, value, key_cache, value_cache,
151 + slot_mapping)
152
153 # Reference implementation.
154 for i in range(num_tokens):
136 - reshaped_key = cloned_key.reshape(num_tokens, num_heads, head_size // x, x)
137 - block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
155 + reshaped_key = cloned_key.reshape(num_tokens, num_heads,
156 + head_size // x, x)
157 + block_idx = torch.div(slot_mapping[i],
158 + block_size,
159 + rounding_mode='floor')
160 block_offset = slot_mapping[i] % block_size
161 reshaped_key[i] = key_cache[block_idx, :, :, block_offset, :]
162 cloned_value[i] = value_cache[block_idx, :, :, block_offset]
@@ -145,20 +167,30 @@ def run_gather_cached_kv(
167
168 def test_copy_blocks() -> None:
169 for dtype in [torch.half, torch.bfloat16, torch.float]:
148 - run_copy_blocks(
149 - num_mappings=23, num_layers=7, num_heads=17, head_size=16,
150 - block_size=8, num_blocks=1024, dtype=dtype)
170 + run_copy_blocks(num_mappings=23,
171 + num_layers=7,
172 + num_heads=17,
173 + head_size=16,
174 + block_size=8,
175 + num_blocks=1024,
176 + dtype=dtype)
177
178
179 def test_reshape_and_cache() -> None:
180 for dtype in [torch.half, torch.bfloat16, torch.float]:
155 - run_reshape_and_cache(
156 - num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
157 - dtype=dtype)
181 + run_reshape_and_cache(num_tokens=3,
182 + num_heads=2,
183 + head_size=16,
184 + block_size=8,
185 + num_blocks=2,
186 + dtype=dtype)
187
188
189 def test_gather_cached_kv() -> None:
190 for dtype in [torch.half, torch.bfloat16, torch.float]:
162 - run_gather_cached_kv(
163 - num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
164 - dtype=dtype)
191 + run_gather_cached_kv(num_tokens=3,
192 + num_heads=2,
193 + head_size=16,
194 + block_size=8,
195 + num_blocks=2,
196 + dtype=dtype)
tests/kernels/test_layernorm.py
+4 -2
@@ -14,8 +14,10 @@ class RefRMSNorm(nn.Module):
14 self.variance_epsilon = eps
15
16 def forward(self, hidden_states):
17 - variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
18 - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
17 + variance = hidden_states.to(torch.float32).pow(2).mean(-1,
18 + keepdim=True)
19 + hidden_states = hidden_states * torch.rsqrt(variance +
20 + self.variance_epsilon)
21 if self.weight.dtype in [torch.half, torch.float16, torch.bfloat16]:
22 hidden_states = hidden_states.to(self.weight.dtype)
23 return self.weight * hidden_states
tests/kernels/test_pos_encoding.py
+20 -15
@@ -8,8 +8,8 @@ from vllm import pos_encoding_ops
8
9
10 def rotate_half(x: torch.Tensor) -> torch.Tensor:
11 - x1 = x[..., : x.shape[-1] // 2]
12 - x2 = x[..., x.shape[-1] // 2 :]
11 + x1 = x[..., :x.shape[-1] // 2]
12 + x2 = x[..., x.shape[-1] // 2:]
13 return torch.cat((-x2, x1), dim=-1)
14
15
@@ -38,7 +38,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
38 self.max_position_embeddings = max_position_embeddings
39
40 # Create cos and sin embeddings.
41 - inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2) / dim))
41 + inv_freq = 1.0 / (base**(torch.arange(0, dim, 2) / dim))
42 t = torch.arange(max_position_embeddings).float()
43 freqs = torch.einsum("i,j->ij", t, inv_freq.float())
44 emb = torch.cat((freqs, freqs), dim=-1)
@@ -49,16 +49,15 @@ class RefRotaryEmbeddingNeox(nn.Module):
49
50 def forward(
51 self,
52 - positions: torch.Tensor, # [num_tokens]
53 - query: torch.Tensor, # [num_tokens, num_heads, head_size]
54 - key: torch.Tensor, # [num_tokens, num_heads, head_size]
52 + positions: torch.Tensor, # [num_tokens]
53 + query: torch.Tensor, # [num_tokens, num_heads, head_size]
54 + key: torch.Tensor, # [num_tokens, num_heads, head_size]
55 ) -> Tuple[torch.Tensor, torch.Tensor]:
56
57 - query_rot = query[..., : self.rotary_dim]
58 - query_pass = query[..., self.rotary_dim :]
59 - key_rot = key[..., : self.rotary_dim]
60 - key_pass = key[..., self.rotary_dim :]
61 -
57 + query_rot = query[..., :self.rotary_dim]
58 + query_pass = query[..., self.rotary_dim:]
59 + key_rot = key[..., :self.rotary_dim]
60 + key_pass = key[..., self.rotary_dim:]
61
62 query_rot = query_rot.transpose(0, 1)
63 key_rot = key_rot.transpose(0, 1)
@@ -85,12 +84,18 @@ def run_rotary_embedding_neox(
84 dtype: torch.dtype,
85 base: int = 10000,
86 ) -> None:
88 - positions = torch.randint(0, max_position, (num_tokens,), device='cuda')
89 - query = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
90 - key = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
87 + positions = torch.randint(0, max_position, (num_tokens, ), device='cuda')
88 + query = torch.randn(num_tokens,
89 + num_heads * head_size,
90 + dtype=dtype,
91 + device='cuda')
92 + key = torch.randn(num_tokens,
93 + num_heads * head_size,
94 + dtype=dtype,
95 + device='cuda')
96
97 # Create the rotary embedding.
93 - inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
98 + inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
99 t = torch.arange(max_position).float()
100 freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
101 cos = freqs.cos()
vllm/__init__.py
+2
@@ -1,3 +1,5 @@
1 +"""vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
2 +
3 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4 from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
vllm/block.py
+2 -1
@@ -35,7 +35,8 @@ class LogicalTokenBlock:
35
36 def append_tokens(self, token_ids: List[int]) -> None:
37 assert len(token_ids) <= self.get_num_empty_slots()
38 - self.token_ids[self.num_tokens:self.num_tokens + len(token_ids)] = token_ids
38 + curr_idx = self.num_tokens
39 + self.token_ids[curr_idx:curr_idx + len(token_ids)] = token_ids
40 self.num_tokens += len(token_ids)
41
42 def get_token_ids(self) -> List[int]:
vllm/config.py
+12 -14
@@ -8,7 +8,7 @@ from vllm.utils import get_cpu_memory
8
9 logger = init_logger(__name__)
10
11 -_GiB = 1 << 30
11 +_GB = 1 << 30
12
13
14 class ModelConfig:
@@ -106,6 +106,7 @@ class CacheConfig:
106 vLLM execution.
107 swap_space: Size of the CPU swap space per GPU (in GiB).
108 """
109 +
110 def __init__(
111 self,
112 block_size: int,
@@ -114,7 +115,7 @@ class CacheConfig:
115 ) -> None:
116 self.block_size = block_size
117 self.gpu_memory_utilization = gpu_memory_utilization
117 - self.swap_space_bytes = swap_space * _GiB
118 + self.swap_space_bytes = swap_space * _GB
119 self._verify_args()
120
121 # Will be set after profiling.
@@ -137,14 +138,13 @@ class CacheConfig:
138 num_gpus_per_node = parallel_config.tensor_parallel_size
139 cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node
140
140 - msg = (
141 - f"{cpu_memory_usage / _GiB:.2f} GiB out of "
142 - f"the {total_cpu_memory / _GiB:.2f} GiB total CPU memory is "
143 - "allocated for the swap space.")
141 + msg = (f"{cpu_memory_usage / _GB:.2f} GiB out of "
142 + f"the {total_cpu_memory / _GB:.2f} GiB total CPU memory is "
143 + "allocated for the swap space.")
144 if cpu_memory_usage > 0.7 * total_cpu_memory:
145 raise ValueError("Too large swap space. " + msg)
146 elif cpu_memory_usage > 0.4 * total_cpu_memory:
147 - logger.warn("Possibly too large swap space. " + msg)
147 + logger.warning("Possibly too large swap space. " + msg)
148
149
150 class ParallelConfig:
@@ -157,6 +157,7 @@ class ParallelConfig:
157 True if either pipeline_parallel_size or tensor_parallel_size is
158 greater than 1.
159 """
160 +
161 def __init__(
162 self,
163 pipeline_parallel_size: int,
@@ -189,12 +190,9 @@ class SchedulerConfig:
190 max_seq_len: Maximum length of a sequence (including prompt
191 and generated text).
192 """
192 - def __init__(
193 - self,
194 - max_num_batched_tokens: int,
195 - max_num_seqs: int,
196 - max_seq_len: int
197 - ) -> None:
193 +
194 + def __init__(self, max_num_batched_tokens: int, max_num_seqs: int,
195 + max_seq_len: int) -> None:
196 self.max_num_batched_tokens = max_num_batched_tokens
197 self.max_num_seqs = max_num_seqs
198 self.max_seq_len = max_seq_len
@@ -241,7 +239,7 @@ def _get_and_verify_dtype(
239 pass
240 else:
241 # Casting between float16 and bfloat16 is allowed with a warning.
244 - logger.warn(f"Casting {config_dtype} to {torch_dtype}.")
242 + logger.warning(f"Casting {config_dtype} to {torch_dtype}.")
243
244 # Check if the GPU supports the dtype.
245 if torch_dtype == torch.bfloat16:
vllm/core/block_manager.py
+9 -5
@@ -27,8 +27,9 @@ class BlockAllocator:
27 # Initialize the free blocks.
28 self.free_blocks: List[PhysicalTokenBlock] = []
29 for i in range(num_blocks):
30 - block = PhysicalTokenBlock(
31 - device=device, block_number=i, block_size=block_size)
30 + block = PhysicalTokenBlock(device=device,
31 + block_number=i,
32 + block_size=block_size)
33 self.free_blocks.append(block)
34
35 def allocate(self) -> PhysicalTokenBlock:
@@ -84,10 +85,12 @@ class BlockSpaceManager:
85 num_required_blocks = len(seq.logical_token_blocks)
86 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
87 # Use watermark to avoid frequent cache eviction.
87 - return num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks
88 + return (num_free_gpu_blocks - num_required_blocks >=
89 + self.watermark_blocks)
90
91 def allocate(self, seq_group: SequenceGroup) -> None:
90 - # NOTE: Here we assume that all sequences in the group have the same prompt.
92 + # NOTE: Here we assume that all sequences in the group have the same
93 + # prompt.
94 seq = seq_group.get_seqs()[0]
95
96 # Allocate new physical token blocks that will store the prompt tokens.
@@ -143,7 +146,8 @@ class BlockSpaceManager:
146 for block in src_block_table:
147 block.ref_count += 1
148
146 - def _get_physical_blocks(self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
149 + def _get_physical_blocks(
150 + self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
151 # NOTE: Here, we assume that the physical blocks are only shared by
152 # the sequences in the same group.
153 blocks: Set[PhysicalTokenBlock] = set()
vllm/core/scheduler.py
+35 -30
@@ -43,8 +43,7 @@ class SchedulerOutputs:
43 assert not (blocks_to_swap_in and blocks_to_swap_out)
44
45 def is_empty(self) -> bool:
46 - return (not self.blocks_to_swap_in
47 - and not self.blocks_to_swap_out
46 + return (not self.blocks_to_swap_in and not self.blocks_to_swap_out
47 and not self.blocks_to_copy)
48
49
@@ -61,7 +60,7 @@ class Scheduler:
60 self.log_stats = log_stats
61
62 # Instantiate the scheduling policy.
64 - self.policy = PolicyFactory.get_policy(policy_name='fcfs')
63 + self.policy = PolicyFactory.get_policy(policy_name="fcfs")
64 # Create the block space manager.
65 self.block_manager = BlockSpaceManager(
66 block_size=self.cache_config.block_size,
@@ -102,7 +101,8 @@ class Scheduler:
101 def get_num_unfinished_seq_groups(self) -> int:
102 return len(self.waiting) + len(self.running) + len(self.swapped)
103
105 - def _schedule(self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
104 + def _schedule(
105 + self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
106 # Blocks that need to be swaped or copied before model execution.
107 blocks_to_swap_in: Dict[int, int] = {}
108 blocks_to_swap_out: Dict[int, int] = {}
@@ -160,7 +160,8 @@ class Scheduler:
160 num_curr_seqs = sum(
161 seq_group.num_seqs(status=SequenceStatus.RUNNING)
162 for seq_group in self.running)
163 - if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
163 + if (num_curr_seqs + num_new_seqs >
164 + self.scheduler_config.max_num_seqs):
165 break
166
167 seq_group = self.swapped.pop(0)
@@ -170,8 +171,7 @@ class Scheduler:
171
172 num_batched_tokens = sum(
173 seq_group.num_seqs(status=SequenceStatus.RUNNING)
173 - for seq_group in self.running
174 - )
174 + for seq_group in self.running)
175
176 # Join waiting sequences if possible.
177 prompt_group_ids: List[str] = []
@@ -191,7 +191,7 @@ class Scheduler:
191
192 num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193 if num_prompt_tokens >= self.scheduler_config.max_seq_len:
194 - logger.warn(
194 + logger.warning(
195 f"Input prompt ({num_prompt_tokens} tokens) is too long"
196 " and exceeds limit of "
197 f"{self.scheduler_config.max_seq_len}")
@@ -206,17 +206,19 @@ class Scheduler:
206 break
207
208 # If the number of batched tokens exceeds the limit, stop.
209 - if (num_batched_tokens + num_prompt_tokens
210 - > self.scheduler_config.max_num_batched_tokens):
209 + if (num_batched_tokens + num_prompt_tokens >
210 + self.scheduler_config.max_num_batched_tokens):
211 break
212
213 # The total number of sequences in the RUNNING state should not
214 # exceed the maximum number of sequences.
215 - num_new_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
215 + num_new_seqs = seq_group.num_seqs(
216 + status=SequenceStatus.WAITING)
217 num_curr_seqs = sum(
218 seq_group.num_seqs(status=SequenceStatus.RUNNING)
219 for seq_group in self.running)
219 - if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
220 + if (num_curr_seqs + num_new_seqs >
221 + self.scheduler_config.max_num_seqs):
222 break
223
224 seq_group = self.waiting.pop(0)
@@ -240,12 +242,11 @@ class Scheduler:
242 elapsed_time = now - self.last_logging_time
243 if elapsed_time > _LOGGING_INTERVAL_SEC:
244 self.last_logging_time = now
243 - self.num_input_tokens = [
244 - (t, n) for t, n in self.num_input_tokens
245 - if now - t < _LOGGING_INTERVAL_SEC
246 - ]
245 + self.num_input_tokens = [(t, n) for t, n in self.num_input_tokens
246 + if now - t < _LOGGING_INTERVAL_SEC]
247 if len(self.num_input_tokens) > 1:
248 - total_num_tokens = sum(n for _, n in self.num_input_tokens[:-1])
248 + total_num_tokens = sum(n
249 + for _, n in self.num_input_tokens[:-1])
250 window = now - self.num_input_tokens[0][0]
251 avg_throughput = total_num_tokens / window
252 else:
@@ -258,26 +259,30 @@ class Scheduler:
259
260 total_num_cpu_blocks = self.cache_config.num_cpu_blocks
261 if total_num_cpu_blocks > 0:
261 - num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
262 + num_free_cpu_blocks = (
263 + self.block_manager.get_num_free_cpu_blocks())
264 num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
265 cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
266 else:
267 cpu_cache_usage = 0.0
268
267 - logger.info(
268 - f"Throughput: {avg_throughput:.1f} tokens/s, "
269 - f"Running: {len(self.running)} reqs, "
270 - f"Swapped: {len(self.swapped)} reqs, "
271 - f"Pending: {len(self.waiting)} reqs, "
272 - f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
273 - f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
269 + logger.info(f"Throughput: {avg_throughput:.1f} tokens/s, "
270 + f"Running: {len(self.running)} reqs, "
271 + f"Swapped: {len(self.swapped)} reqs, "
272 + f"Pending: {len(self.waiting)} reqs, "
273 + f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
274 + f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
275 return scheduler_outputs, prompt_group_ids, ignored_seq_groups
276
276 - def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, List[SequenceGroup]]:
277 + def schedule(
278 + self
279 + ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
280 + List[SequenceGroup]]:
281 # Schedule sequence groups.
282 # This function call changes the internal states of the scheduler
283 # such as self.running, self.swapped, and self.waiting.
280 - scheduler_outputs, prompt_group_ids, ignored_seq_groups = self._schedule()
284 + (scheduler_outputs, prompt_group_ids,
285 + ignored_seq_groups) = self._schedule()
286
287 # Create input data structures.
288 seq_group_metadata_list: List[SequenceGroupMetadata] = []
@@ -311,8 +316,8 @@ class Scheduler:
316 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
317 output = seq_outputs[seq.seq_id]
318 if seq.seq_id != output.parent_seq_id:
314 - # The sequence is a fork of the parent sequence (beam search).
315 - # Free the current sequence.
319 + # The sequence is a fork of the parent sequence (beam
320 + # search). Free the current sequence.
321 self.block_manager.free(seq)
322 # Fork the parent sequence.
323 parent_seq = seq_group.find(output.parent_seq_id)
@@ -385,7 +390,7 @@ class Scheduler:
390 elif preemption_mode == PreemptionMode.SWAP:
391 self._preempt_by_swap(seq_group, blocks_to_swap_out)
392 else:
388 - assert False, 'Invalid preemption mode.'
393 + assert False, "Invalid preemption mode."
394
395 def _preempt_by_recompute(
396 self,
vllm/engine/arg_utils.py
+78 -51
@@ -12,11 +12,11 @@ class EngineArgs:
12 """Arguments for vLLM engine."""
13 model: str
14 tokenizer: Optional[str] = None
15 - tokenizer_mode: str = "auto"
15 + tokenizer_mode: str = 'auto'
16 download_dir: Optional[str] = None
17 use_np_weights: bool = False
18 use_dummy_weights: bool = False
19 - dtype: str = "auto"
19 + dtype: str = 'auto'
20 seed: int = 0
21 worker_use_ray: bool = False
22 pipeline_parallel_size: int = 1
@@ -35,76 +35,101 @@ class EngineArgs:
35
36 @staticmethod
37 def add_cli_args(
38 - parser: argparse.ArgumentParser,
39 - ) -> argparse.ArgumentParser:
38 + parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
39 """Shared CLI arguments for vLLM engine."""
40 # Model arguments
42 - parser.add_argument('--model', type=str, default='facebook/opt-125m',
43 - help='name or path of the huggingface model to use')
44 - parser.add_argument('--tokenizer', type=str, default=EngineArgs.tokenizer,
45 - help='name or path of the huggingface tokenizer to use')
46 - parser.add_argument('--tokenizer-mode', type=str,
41 + parser.add_argument(
42 + '--model',
43 + type=str,
44 + default='facebook/opt-125m',
45 + help='name or path of the huggingface model to use')
46 + parser.add_argument(
47 + '--tokenizer',
48 + type=str,
49 + default=EngineArgs.tokenizer,
50 + help='name or path of the huggingface tokenizer to use')
51 + parser.add_argument('--tokenizer-mode',
52 + type=str,
53 default=EngineArgs.tokenizer_mode,
54 choices=['auto', 'slow'],
55 help='tokenizer mode. "auto" will use the fast '
50 - 'tokenizer if available, and "slow" will '
51 - 'always use the slow tokenizer.')
52 - parser.add_argument('--download-dir', type=str,
56 + 'tokenizer if available, and "slow" will '
57 + 'always use the slow tokenizer.')
58 + parser.add_argument('--download-dir',
59 + type=str,
60 default=EngineArgs.download_dir,
61 help='directory to download and load the weights, '
55 - 'default to the default cache dir of '
56 - 'huggingface')
57 - parser.add_argument('--use-np-weights', action='store_true',
62 + 'default to the default cache dir of '
63 + 'huggingface')
64 + parser.add_argument('--use-np-weights',
65 + action='store_true',
66 help='save a numpy copy of model weights for '
59 - 'faster loading. This can increase the disk '
60 - 'usage by up to 2x.')
61 - parser.add_argument('--use-dummy-weights', action='store_true',
67 + 'faster loading. This can increase the disk '
68 + 'usage by up to 2x.')
69 + parser.add_argument('--use-dummy-weights',
70 + action='store_true',
71 help='use dummy values for model weights')
72 # TODO(woosuk): Support FP32.
64 - parser.add_argument('--dtype', type=str, default=EngineArgs.dtype,
65 - choices=['auto', 'half', 'bfloat16', 'float'],
66 - help='data type for model weights and activations. '
67 - 'The "auto" option will use FP16 precision '
68 - 'for FP32 and FP16 models, and BF16 precision '
69 - 'for BF16 models.')
73 + parser.add_argument(
74 + '--dtype',
75 + type=str,
76 + default=EngineArgs.dtype,
77 + choices=['auto', 'half', 'bfloat16', 'float'],
78 + help='data type for model weights and activations. '
79 + 'The "auto" option will use FP16 precision '
80 + 'for FP32 and FP16 models, and BF16 precision '
81 + 'for BF16 models.')
82 # Parallel arguments
71 - parser.add_argument('--worker-use-ray', action='store_true',
83 + parser.add_argument('--worker-use-ray',
84 + action='store_true',
85 help='use Ray for distributed serving, will be '
73 - 'automatically set when using more than 1 GPU')
74 - parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
86 + 'automatically set when using more than 1 GPU')
87 + parser.add_argument('--pipeline-parallel-size',
88 + '-pp',
89 + type=int,
90 default=EngineArgs.pipeline_parallel_size,
91 help='number of pipeline stages')
77 - parser.add_argument('--tensor-parallel-size', '-tp', type=int,
92 + parser.add_argument('--tensor-parallel-size',
93 + '-tp',
94 + type=int,
95 default=EngineArgs.tensor_parallel_size,
96 help='number of tensor parallel replicas')
97 # KV cache arguments
81 - parser.add_argument('--block-size', type=int,
98 + parser.add_argument('--block-size',
99 + type=int,
100 default=EngineArgs.block_size,
101 choices=[8, 16, 32],
102 help='token block size')
103 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
86 - parser.add_argument('--seed', type=int, default=EngineArgs.seed,
104 + parser.add_argument('--seed',
105 + type=int,
106 + default=EngineArgs.seed,
107 help='random seed')
88 - parser.add_argument('--swap-space', type=int,
108 + parser.add_argument('--swap-space',
109 + type=int,
110 default=EngineArgs.swap_space,
111 help='CPU swap space size (GiB) per GPU')
91 - parser.add_argument('--gpu-memory-utilization', type=float,
112 + parser.add_argument('--gpu-memory-utilization',
113 + type=float,
114 default=EngineArgs.gpu_memory_utilization,
115 help='the percentage of GPU memory to be used for'
94 - 'the model executor')
95 - parser.add_argument('--max-num-batched-tokens', type=int,
116 + 'the model executor')
117 + parser.add_argument('--max-num-batched-tokens',
118 + type=int,
119 default=EngineArgs.max_num_batched_tokens,
120 help='maximum number of batched tokens per '
98 - 'iteration')
99 - parser.add_argument('--max-num-seqs', type=int,
121 + 'iteration')
122 + parser.add_argument('--max-num-seqs',
123 + type=int,
124 default=EngineArgs.max_num_seqs,
125 help='maximum number of sequences per iteration')
102 - parser.add_argument('--disable-log-stats', action='store_true',
126 + parser.add_argument('--disable-log-stats',
127 + action='store_true',
128 help='disable logging statistics')
129 return parser
130
131 @classmethod
107 - def from_cli_args(cls, args: argparse.Namespace) -> "EngineArgs":
132 + def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs':
133 # Get the list of attributes of this dataclass.
134 attrs = [attr.name for attr in dataclasses.fields(cls)]
135 # Set the attributes from the parsed arguments.
@@ -115,18 +140,19 @@ class EngineArgs:
140 self,
141 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
142 # Initialize the configs.
118 - model_config = ModelConfig(
119 - self.model, self.tokenizer, self.tokenizer_mode, self.download_dir,
120 - self.use_np_weights, self.use_dummy_weights, self.dtype, self.seed)
121 - cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
143 + model_config = ModelConfig(self.model, self.tokenizer,
144 + self.tokenizer_mode, self.download_dir,
145 + self.use_np_weights, self.use_dummy_weights,
146 + self.dtype, self.seed)
147 + cache_config = CacheConfig(self.block_size,
148 + self.gpu_memory_utilization,
149 self.swap_space)
150 parallel_config = ParallelConfig(self.pipeline_parallel_size,
151 self.tensor_parallel_size,
152 self.worker_use_ray)
126 - max_seq_len = min(
127 - self.max_num_batched_tokens,
128 - getattr(model_config.hf_config, "max_position_embeddings",
129 - float("inf")))
153 + model_max_len = getattr(model_config.hf_config,
154 + 'max_position_embeddings', float('inf'))
155 + max_seq_len = min(self.max_num_batched_tokens, model_max_len)
156 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
157 self.max_num_seqs, max_seq_len)
158 return model_config, cache_config, parallel_config, scheduler_config
@@ -140,12 +166,13 @@ class AsyncEngineArgs(EngineArgs):
166
167 @staticmethod
168 def add_cli_args(
143 - parser: argparse.ArgumentParser,
144 - ) -> argparse.ArgumentParser:
169 + parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
170 parser = EngineArgs.add_cli_args(parser)
146 - parser.add_argument('--engine-use-ray', action='store_true',
171 + parser.add_argument('--engine-use-ray',
172 + action='store_true',
173 help='use Ray to start the LLM engine in a '
148 - 'separate process as the server process.')
149 - parser.add_argument('--disable-log-requests', action='store_true',
174 + 'separate process as the server process.')
175 + parser.add_argument('--disable-log-requests',
176 + action='store_true',
177 help='disable logging requests')
178 return parser
vllm/engine/async_llm_engine.py
+26 -17
@@ -11,7 +11,7 @@ from vllm.sampling_params import SamplingParams
11
12 logger = init_logger(__name__)
13
14 -TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
14 +TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17 class AsyncLLMEngine:
@@ -35,8 +35,13 @@ class AsyncLLMEngine:
35 log_requests: Whether to log the requests.
36 *args, *kwargs: Arguments for LLMEngine.
37 """
38 - def __init__(self, worker_use_ray: bool, engine_use_ray: bool,
39 - log_requests: bool = True, *args, **kwargs) -> None:
38 +
39 + def __init__(self,
40 + worker_use_ray: bool,
41 + engine_use_ray: bool,
42 + *args,
43 + log_requests: bool = True,
44 + **kwargs) -> None:
45 self.worker_use_ray = worker_use_ray
46 self.engine_use_ray = engine_use_ray
47 self.log_requests = log_requests
@@ -76,12 +81,11 @@ class AsyncLLMEngine:
81 self.request_events[request_id].set()
82
83 async def generate(
79 - self,
80 - prompt: Optional[str],
81 - sampling_params: SamplingParams,
82 - request_id: str,
83 - prompt_token_ids: Optional[List[int]] = None
84 - ) -> RequestOutput:
84 + self,
85 + prompt: Optional[str],
86 + sampling_params: SamplingParams,
87 + request_id: str,
88 + prompt_token_ids: Optional[List[int]] = None) -> RequestOutput:
89 """Generate outputs for a request.
90
91 Generate outputs for a request. This method is a coroutine. It adds the
@@ -117,14 +121,17 @@ class AsyncLLMEngine:
121 # Add the request into the vLLM engine's waiting queue.
122 if self.engine_use_ray:
123 await self.engine.add_request.remote(
120 - request_id, prompt, sampling_params,
124 + request_id,
125 + prompt,
126 + sampling_params,
127 prompt_token_ids=prompt_token_ids,
128 arrival_time=arrival_time)
129 else:
124 - self.engine.add_request(
125 - request_id, prompt, sampling_params,
126 - prompt_token_ids=prompt_token_ids,
127 - arrival_time=arrival_time)
130 + self.engine.add_request(request_id,
131 + prompt,
132 + sampling_params,
133 + prompt_token_ids=prompt_token_ids,
134 + arrival_time=arrival_time)
135
136 # The vLLM engine does not have a background loop that keeps
137 # processing incoming requests. Therefore, we need to keep kicking
@@ -200,7 +207,8 @@ class AsyncLLMEngine:
207 self.kicking_request_id = None
208
209 @classmethod
203 - def from_engine_args(cls, engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
210 + def from_engine_args(cls,
211 + engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
212 """Creates an async LLM engine from the engine arguments."""
213 # Create the engine configs.
214 engine_configs = engine_args.create_engine_configs()
@@ -211,8 +219,9 @@ class AsyncLLMEngine:
219 # Create the async LLM engine.
220 engine = cls(engine_args.worker_use_ray,
221 engine_args.engine_use_ray,
214 - not engine_args.disable_log_requests,
222 *engine_configs,
216 - distributed_init_method, devices,
223 + distributed_init_method,
224 + devices,
225 + log_requests=not engine_args.disable_log_requests,
226 log_stats=not engine_args.disable_log_stats)
227 return engine
vllm/engine/llm_engine.py
+18 -15
@@ -67,8 +67,7 @@ class LLMEngine:
67 f"download_dir={model_config.download_dir!r}, "
68 f"use_np_weights={model_config.use_np_weights}, "
69 f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
70 - f"seed={model_config.seed})"
71 - )
70 + f"seed={model_config.seed})")
71 # TODO(woosuk): Print more configs in debug mode.
72
73 self.model_config = model_config
@@ -78,8 +77,8 @@ class LLMEngine:
77 self.log_stats = log_stats
78 self._verify_args()
79
81 - self.tokenizer = get_tokenizer(model_config.tokenizer,
82 - model_config.tokenizer_mode)
80 + self.tokenizer = get_tokenizer(
81 + model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
82 self.seq_counter = Counter()
83
84 # Create the parallel GPU workers.
@@ -129,8 +128,8 @@ class LLMEngine:
128 num_gpu_blocks = min(b[0] for b in num_blocks)
129 num_cpu_blocks = min(b[1] for b in num_blocks)
130 # FIXME(woosuk): Change to debug log.
132 - logger.info(f'# GPU blocks: {num_gpu_blocks}, '
133 - f'# CPU blocks: {num_cpu_blocks}')
131 + logger.info(f"# GPU blocks: {num_gpu_blocks}, "
132 + f"# CPU blocks: {num_cpu_blocks}")
133
134 if num_gpu_blocks <= 0:
135 raise ValueError("No available memory for the cache blocks. "
@@ -152,7 +151,9 @@ class LLMEngine:
151 # Initialize the cluster.
152 distributed_init_method, devices = initialize_cluster(parallel_config)
153 # Create the LLM engine.
155 - engine = cls(*engine_configs, distributed_init_method, devices,
154 + engine = cls(*engine_configs,
155 + distributed_init_method,
156 + devices,
157 log_stats=not engine_args.disable_log_stats)
158 return engine
159
@@ -226,8 +227,10 @@ class LLMEngine:
227 and updates the scheduler with the model outputs. Finally, it decodes
228 the sequences and returns the newly generated results.
229 """
229 - seq_group_metadata_list, scheduler_outputs, ignored_seq_groups = self.scheduler.schedule()
230 - if (not seq_group_metadata_list) and scheduler_outputs.is_empty() and (not ignored_seq_groups):
230 + (seq_group_metadata_list, scheduler_outputs,
231 + ignored_seq_groups) = self.scheduler.schedule()
232 + if ((not seq_group_metadata_list) and scheduler_outputs.is_empty()
233 + and (not ignored_seq_groups)):
234 # Nothing to do.
235 return []
236
@@ -281,8 +284,8 @@ class LLMEngine:
284 # Truncate the output text so that the stop string is
285 # not included in the output.
286 seq.output_text = seq.output_text[:-len(stop_str)]
284 - self.scheduler.free_seq(seq,
285 - SequenceStatus.FINISHED_STOPPED)
287 + self.scheduler.free_seq(
288 + seq, SequenceStatus.FINISHED_STOPPED)
289 stopped = True
290 break
291 if stopped:
@@ -290,7 +293,7 @@ class LLMEngine:
293
294 # Check if the sequence has reached max_seq_len.
295 if (seq.get_len() >=
293 - self.scheduler.scheduler_config.max_seq_len):
296 + self.scheduler.scheduler_config.max_seq_len):
297 self.scheduler.free_seq(
298 seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
299 continue
@@ -302,15 +305,15 @@ class LLMEngine:
305 # Check if the sequence has generated the EOS token.
306 if not sampling_params.ignore_eos:
307 if seq.get_last_token_id() == self.tokenizer.eos_token_id:
305 - self.scheduler.free_seq(seq,
306 - SequenceStatus.FINISHED_STOPPED)
308 + self.scheduler.free_seq(
309 + seq, SequenceStatus.FINISHED_STOPPED)
310 continue
311
312 def _run_workers(
313 self,
314 method: str,
312 - get_all_outputs: bool = False,
315 *args,
316 + get_all_outputs: bool = False,
317 **kwargs,
318 ) -> Any:
319 """Runs the given method on all workers."""
vllm/engine/ray_utils.py
+7 -6
@@ -8,7 +8,8 @@ except ImportError:
8
9 from vllm.config import ParallelConfig
10
11 -DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), device id
11 +# rank, node resource (node IP), device id
12 +DeviceID = Tuple[int, Optional[str], int]
13
14
15 def initialize_cluster(
@@ -53,15 +54,15 @@ def initialize_cluster(
54 valid_node_resources = []
55 num_devices_per_node = None
56 for node in ray.nodes():
56 - if (not node['Alive']) or node['Resources']['GPU'] <= 0:
57 + if (not node["Alive"]) or node["Resources"]["GPU"] <= 0:
58 continue
59 if num_devices_per_node is None:
59 - num_devices_per_node = node['Resources']['GPU']
60 + num_devices_per_node = node["Resources"]["GPU"]
61 else:
61 - assert num_devices_per_node == node['Resources']['GPU'], (
62 + assert num_devices_per_node == node["Resources"]["GPU"], (
63 "The number of GPUs per node is not uniform.")
63 - for key in node['Resources']:
64 - if key.startswith('node:'):
64 + for key in node["Resources"]:
65 + if key.startswith("node:"):
66 valid_node_resources.append(key)
67
68 # Verify the parallel config.
vllm/entrypoints/api_server.py
+8 -9
@@ -11,8 +11,8 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
11 from vllm.sampling_params import SamplingParams
12 from vllm.utils import random_uuid
13
14 -TIMEOUT_KEEP_ALIVE = 5 # seconds.
15 -TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
14 +TIMEOUT_KEEP_ALIVE = 5 # seconds.
15 +TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds.
16 app = FastAPI()
17
18
@@ -37,8 +37,7 @@ async def generate(request: Request) -> Response:
37 async for request_output in results_generator:
38 prompt = request_output.prompt
39 text_outputs = [
40 - prompt + output.text
41 - for output in request_output.outputs
40 + prompt + output.text for output in request_output.outputs
41 ]
42 ret = {"text": text_outputs}
43 yield (json.dumps(ret) + "\0").encode("utf-8")
@@ -63,10 +62,7 @@ async def generate(request: Request) -> Response:
62
63 assert final_output is not None
64 prompt = final_output.prompt
66 - text_outputs = [
67 - prompt + output.text
68 - for output in final_output.outputs
69 - ]
65 + text_outputs = [prompt + output.text for output in final_output.outputs]
66 ret = {"text": text_outputs}
67 return Response(content=json.dumps(ret))
68
@@ -81,5 +77,8 @@ if __name__ == "__main__":
77 engine_args = AsyncEngineArgs.from_cli_args(args)
78 engine = AsyncLLMEngine.from_engine_args(engine_args)
79
84 - uvicorn.run(app, host=args.host, port=args.port, log_level="debug",
80 + uvicorn.run(app,
81 + host=args.host,
82 + port=args.port,
83 + log_level="debug",
84 timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
vllm/entrypoints/llm.py
+1 -2
@@ -63,8 +63,7 @@ class LLM:
63 self.request_counter = Counter()
64
65 def get_tokenizer(
66 - self,
67 - ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
66 + self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
67 return self.llm_engine.tokenizer
68
69 def set_tokenizer(
vllm/entrypoints/openai/api_server.py
+67 -46
@@ -1,4 +1,5 @@
1 -# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
1 +# Adapted from
2 +# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
3
4 import argparse
5 from http import HTTPStatus
@@ -29,7 +30,7 @@ from vllm.sampling_params import SamplingParams
30 from vllm.transformers_utils.tokenizer import get_tokenizer
31 from vllm.utils import random_uuid
32
32 -TIMEOUT_KEEP_ALIVE = 5 # seconds
33 +TIMEOUT_KEEP_ALIVE = 5 # seconds
34
35 logger = init_logger(__name__)
36 served_model = None
@@ -38,14 +39,13 @@ app = fastapi.FastAPI()
39
40 def create_error_response(status_code: HTTPStatus,
41 message: str) -> JSONResponse:
41 - return JSONResponse(
42 - ErrorResponse(message=message, type="invalid_request_error").dict(),
43 - status_code=status_code.value
44 - )
42 + return JSONResponse(ErrorResponse(message=message,
43 + type="invalid_request_error").dict(),
44 + status_code=status_code.value)
45
46
47 @app.exception_handler(RequestValidationError)
48 -async def validation_exception_handler(request, exc):
48 +async def validation_exception_handler(request, exc): # pylint: disable=unused-argument
49 return create_error_response(HTTPStatus.BAD_REQUEST, str(exc))
50
51
@@ -126,8 +126,11 @@ async def check_length(request, prompt, engine):
126 @app.get("/v1/models")
127 async def show_available_models():
128 """Show available models. Right now we only have one model."""
129 - model_cards = [ModelCard(id=served_model, root=served_model,
130 - permission=[ModelPermission()])]
129 + model_cards = [
130 + ModelCard(id=served_model,
131 + root=served_model,
132 + permission=[ModelPermission()])
133 + ]
134 return ModelList(data=model_cards)
135
136
@@ -144,12 +147,14 @@ def create_logprobs(token_ids: List[int],
147 if len(logprobs.text_offset) == 0:
148 logprobs.text_offset.append(initial_text_offset)
149 else:
147 - logprobs.text_offset.append(logprobs.text_offset[-1] + last_token_len)
150 + logprobs.text_offset.append(logprobs.text_offset[-1] +
151 + last_token_len)
152 last_token_len = len(token)
153
150 - logprobs.top_logprobs.append(
151 - {tokenizer.convert_ids_to_tokens(i): p
152 - for i, p in id_logprob.items()})
154 + logprobs.top_logprobs.append({
155 + tokenizer.convert_ids_to_tokens(i): p
156 + for i, p in id_logprob.items()
157 + })
158 return logprobs
159
160
@@ -348,7 +353,7 @@ async def create_completion(raw_request: Request):
353 if request.suffix is not None:
354 # The language models we currently support do not support suffix.
355 return create_error_response(HTTPStatus.BAD_REQUEST,
351 - "suffix is not currently supported")
356 + "suffix is not currently supported")
357
358 if request.logit_bias is not None:
359 # TODO: support logit_bias in vLLM engine.
@@ -387,22 +392,23 @@ async def create_completion(raw_request: Request):
392 except ValueError as e:
393 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
394
390 - result_generator = engine.generate(prompt, sampling_params,
391 - request_id)
395 + result_generator = engine.generate(prompt, sampling_params, request_id)
396
397 # Similar to the OpenAI API, when n != best_of, we do not stream the
398 # results. In addition, we do not stream the results when use beam search.
395 - stream = (request.stream and
396 - (request.best_of is None or request.n == request.best_of) and
397 - not request.use_beam_search)
399 + stream = (request.stream
400 + and (request.best_of is None or request.n == request.best_of)
401 + and not request.use_beam_search)
402
403 async def abort_request() -> None:
404 await engine.abort(request_id)
405
402 - def create_stream_response_json(index: int,
403 - text: str,
404 - logprobs: Optional[LogProbs] = None,
405 - finish_reason: Optional[str] = None) -> str:
406 + def create_stream_response_json(
407 + index: int,
408 + text: str,
409 + logprobs: Optional[LogProbs] = None,
410 + finish_reason: Optional[str] = None,
411 + ) -> str:
412 choice_data = CompletionResponseStreamChoice(
413 index=index,
414 text=text,
@@ -443,7 +449,8 @@ async def create_completion(raw_request: Request):
449 )
450 yield f"data: {response_json}\n\n"
451 if output.finish_reason is not None:
446 - logprobs = LogProbs() if request.logprobs is not None else None
452 + logprobs = (LogProbs()
453 + if request.logprobs is not None else None)
454 response_json = create_stream_response_json(
455 index=i,
456 text="",
@@ -487,8 +494,8 @@ async def create_completion(raw_request: Request):
494 choices.append(choice_data)
495
496 num_prompt_tokens = len(final_res.prompt_token_ids)
490 - num_generated_tokens = sum(len(output.token_ids)
491 - for output in final_res.outputs)
497 + num_generated_tokens = sum(
498 + len(output.token_ids) for output in final_res.outputs)
499 usage = UsageInfo(
500 prompt_tokens=num_prompt_tokens,
501 completion_tokens=num_generated_tokens,
@@ -506,9 +513,11 @@ async def create_completion(raw_request: Request):
513 # When user requests streaming but we don't stream, we still need to
514 # return a streaming response with a single event.
515 response_json = response.json(ensure_ascii=False)
516 +
517 async def fake_stream_generator() -> AsyncGenerator[str, None]:
518 yield f"data: {response_json}\n\n"
519 yield "data: [DONE]\n\n"
520 +
521 return StreamingResponse(fake_stream_generator(),
522 media_type="text/event-stream")
523
@@ -517,26 +526,34 @@ async def create_completion(raw_request: Request):
526
527 if __name__ == "__main__":
528 parser = argparse.ArgumentParser(
520 - description="vLLM OpenAI-Compatible RESTful API server."
521 - )
522 - parser.add_argument("--host", type=str, default="localhost", help="host name")
529 + description="vLLM OpenAI-Compatible RESTful API server.")
530 + parser.add_argument("--host",
531 + type=str,
532 + default="localhost",
533 + help="host name")
534 parser.add_argument("--port", type=int, default=8000, help="port number")
535 + parser.add_argument("--allow-credentials",
536 + action="store_true",
537 + help="allow credentials")
538 + parser.add_argument("--allowed-origins",
539 + type=json.loads,
540 + default=["*"],
541 + help="allowed origins")
542 + parser.add_argument("--allowed-methods",
543 + type=json.loads,
544 + default=["*"],
545 + help="allowed methods")
546 + parser.add_argument("--allowed-headers",
547 + type=json.loads,
548 + default=["*"],
549 + help="allowed headers")
550 parser.add_argument(
525 - "--allow-credentials", action="store_true", help="allow credentials"
526 - )
527 - parser.add_argument(
528 - "--allowed-origins", type=json.loads, default=["*"], help="allowed origins"
529 - )
530 - parser.add_argument(
531 - "--allowed-methods", type=json.loads, default=["*"], help="allowed methods"
532 - )
533 - parser.add_argument(
534 - "--allowed-headers", type=json.loads, default=["*"], help="allowed headers"
535 - )
536 - parser.add_argument("--served-model-name", type=str, default=None,
537 - help="The model name used in the API. If not specified, "
538 - "the model name will be the same as the "
539 - "huggingface name.")
551 + "--served-model-name",
552 + type=str,
553 + default=None,
554 + help="The model name used in the API. If not specified, "
555 + "the model name will be the same as the "
556 + "huggingface name.")
557 parser = AsyncEngineArgs.add_cli_args(parser)
558 args = parser.parse_args()
559
@@ -556,7 +573,11 @@ if __name__ == "__main__":
573 engine = AsyncLLMEngine.from_engine_args(engine_args)
574
575 # A separate tokenizer to map token IDs to strings.
559 - tokenizer = get_tokenizer(engine_args.tokenizer, engine_args.tokenizer_mode)
576 + tokenizer = get_tokenizer(engine_args.tokenizer,
577 + tokenizer_mode=engine_args.tokenizer_mode)
578
561 - uvicorn.run(app, host=args.host, port=args.port, log_level="info",
579 + uvicorn.run(app,
580 + host=args.host,
581 + port=args.port,
582 + log_level="info",
583 timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
vllm/entrypoints/openai/protocol.py
+4 -2
@@ -1,4 +1,5 @@
1 -# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
1 +# Adapted from
2 +# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3 import time
4 from typing import Dict, List, Literal, Optional, Union
5
@@ -98,7 +99,8 @@ class LogProbs(BaseModel):
99 text_offset: List[int] = Field(default_factory=list)
100 token_logprobs: List[Optional[float]] = Field(default_factory=list)
101 tokens: List[str] = Field(default_factory=list)
101 - top_logprobs: List[Optional[Dict[str, float]]] = Field(default_factory=list)
102 + top_logprobs: List[Optional[Dict[str,
103 + float]]] = Field(default_factory=list)
104
105
106 class CompletionResponseChoice(BaseModel):
vllm/logger.py
+3 -3
@@ -1,9 +1,9 @@
1 -# Adapted from https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
2 -
1 +# Adapted from
2 +# https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
3 +"""Logging configuration for vLLM."""
4 import logging
5 import sys
6
6 -
7 _FORMAT = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s"
8 _DATE_FORMAT = "%m-%d %H:%M:%S"
9
vllm/model_executor/__init__.py
-1
@@ -2,7 +2,6 @@ from vllm.model_executor.input_metadata import InputMetadata
2 from vllm.model_executor.model_loader import get_model
3 from vllm.model_executor.utils import set_random_seed
4
5 -
5 __all__ = [
6 "InputMetadata",
7 "get_model",
vllm/model_executor/input_metadata.py
+13 -2
@@ -8,11 +8,22 @@ from vllm.sequence import SequenceData
8
9
10 class InputMetadata:
11 + """Metadata for input sequences. Used for PagedAttention.
12 +
13 + Args:
14 + seq_groups: List of (seq_ids, sampling_params).
15 + seq_data: Seq_id -> SequenceData.
16 + prompt_lens: Lengths of prompts.
17 + slot_mapping: The address to write the new KV to of each token.
18 + context_lens: the length of attention context for each generation token.
19 + max_context_len: The maximum context length.
20 + block_tables: The block tables. (Seq id -> list of physical block)
21 + """
22
23 def __init__(
24 self,
14 - seq_groups: List[Tuple[List[int], SamplingParams]], # List of (seq_ids, sampling_params).
15 - seq_data: Dict[int, SequenceData], # Seq_id -> SequenceData.
25 + seq_groups: List[Tuple[List[int], SamplingParams]],
26 + seq_data: Dict[int, SequenceData],
27 prompt_lens: List[int],
28 slot_mapping: torch.Tensor,
29 context_lens: torch.Tensor,
vllm/model_executor/layers/activation.py
+9 -10
@@ -6,9 +6,10 @@ from vllm import activation_ops
6
7 _ACTIVATION_REGISTRY = {
8 "gelu": nn.GELU(),
9 - "gelu_new": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
10 - "gelu_fast": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
11 - "gelu_pytorch_tanh": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
9 + # NOTE: The following GELU functions may introduce small rounding errors.
10 + "gelu_new": nn.GELU(approximate="tanh"),
11 + "gelu_fast": nn.GELU(approximate="tanh"),
12 + "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
13 "relu": nn.ReLU(),
14 }
15
@@ -25,15 +26,13 @@ class SiluAndMul(nn.Module):
26 """An activation function for SwiGLU.
27
28 The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[1] // 2.
28 - """
29
30 - def __init__(self):
31 - super().__init__()
30 + Shapes:
31 + x: (num_tokens, 2 * d)
32 + return: (num_tokens, d)
33 + """
34
33 - def forward(
34 - self,
35 - x: torch.Tensor, # (num_tokens, 2 * d)
36 - ) -> torch.Tensor: # (num_tokens, d)
35 + def forward(self, x: torch.Tensor) -> torch.Tensor:
36 num_tokens = x.shape[0]
37 d = x.shape[1] // 2
38 out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
vllm/model_executor/layers/attention.py
+81 -32
@@ -14,6 +14,7 @@ _SUPPORTED_HEAD_SIZES = [64, 80, 96, 128]
14
15
16 class PagedAttention(nn.Module):
17 + # pylint: disable=line-too-long
18 """GPT-style multi-head PagedAttention.
19
20 This class takes flattened 1D query, key, and value tensors as input. The
@@ -54,12 +55,20 @@ class PagedAttention(nn.Module):
55
56 def multi_query_kv_attention(
57 self,
57 - output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
58 - query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
59 - key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
60 - value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
58 + output: torch.Tensor,
59 + query: torch.Tensor,
60 + key: torch.Tensor,
61 + value: torch.Tensor,
62 attn_bias: xops.AttentionBias,
63 ) -> torch.Tensor:
64 + """Normal attention for the prompt tokens.
65 +
66 + Args:
67 + output: shape = [num_prompt_tokens, num_heads, head_size]
68 + query: shape = [num_prompt_tokens, num_heads, head_size]
69 + key: shape = [num_prompt_tokens, num_heads, head_size]
70 + value: shape = [num_prompt_tokens, num_heads, head_size]
71 + """
72 # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
73 out = xops.memory_efficient_attention_forward(
74 query.unsqueeze(0),
@@ -76,12 +85,22 @@ class PagedAttention(nn.Module):
85
86 def single_query_cached_kv_attention(
87 self,
79 - output: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
80 - query: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
81 - key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
82 - value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
88 + output: torch.Tensor,
89 + query: torch.Tensor,
90 + key_cache: torch.Tensor,
91 + value_cache: torch.Tensor,
92 input_metadata: InputMetadata,
93 ) -> None:
94 + """PagedAttention for the generation tokens.
95 +
96 + Args:
97 + output: shape = [num_generation_tokens, num_heads, head_size]
98 + query: shape = [num_generation_tokens, num_heads, head_size]
99 + key_cache: shape = [num_blocks, num_heads, head_size/x,
100 + block_size, x]
101 + value_cache: shape = [num_blocks, num_heads, head_size, block_size]
102 + input_metadata: metadata for paged attention.
103 + """
104 block_size = value_cache.shape[3]
105 attention_ops.single_query_cached_kv_attention(
106 output,
@@ -97,16 +116,32 @@ class PagedAttention(nn.Module):
116
117 def forward(
118 self,
100 - query: torch.Tensor, # [num_tokens, num_heads * head_size]
101 - key: torch.Tensor, # [num_tokens, num_heads * head_size]
102 - value: torch.Tensor, # [num_tokens, num_heads * head_size]
103 - key_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size/x, block_size, x]
104 - value_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size, block_size]
119 + query: torch.Tensor,
120 + key: torch.Tensor,
121 + value: torch.Tensor,
122 + key_cache: Optional[torch.Tensor],
123 + value_cache: Optional[torch.Tensor],
124 input_metadata: InputMetadata,
125 cache_event: Optional[torch.cuda.Event],
107 - ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
108 - # NOTE: The query, key, and value tensors must be sliced from a qkv
109 - # tensor of shape [num_tokens, 3 * num_heads * head_size].
126 + ) -> torch.Tensor:
127 + """PagedAttention forward pass.
128 +
129 + NOTE: The query, key, and value tensors must be sliced from a qkv
130 + tensor of shape [num_tokens, 3 * num_heads * head_size].
131 +
132 + Args:
133 + query: shape = [num_tokens, num_heads * head_size]
134 + key: shape = [num_tokens, num_heads * head_size]
135 + value: shape = [num_tokens, num_heads * head_size]
136 + key_cache: shape = [num_blocks, num_heads, head_size/x,
137 + block_size, x]
138 + value_cache: shape = [num_blocks, num_heads, head_size, block_size]
139 + input_metadata: metadata for paged attention.
140 + cache_event: event to wait for the cache operations to finish.
141 +
142 + Returns:
143 + shape = [num_tokens, num_heads * head_size]
144 + """
145
146 # Reshape the query, key, and value tensors.
147 query = query.view(-1, self.num_heads, self.head_size)
@@ -136,7 +171,7 @@ class PagedAttention(nn.Module):
171 # and value vectors will not be cached.
172 num_valid_tokens = input_metadata.num_valid_tokens
173 if (num_valid_tokens > 0 and key_cache is not None
139 - and value_cache is not None):
174 + and value_cache is not None):
175 # The stride is 3 because the key and value are sliced from qkv.
176 cache_ops.reshape_and_cache(
177 key[:num_valid_tokens],
@@ -149,15 +184,12 @@ class PagedAttention(nn.Module):
184 if input_metadata.num_generation_tokens > 0:
185 assert key_cache is not None and value_cache is not None, (
186 "key_cache and value_cache must be provided when "
152 - "generating tokens."
153 - )
187 + "generating tokens.")
188 # Compute the attention op for generation tokens.
189 self.single_query_cached_kv_attention(
190 output[num_prompt_tokens:num_valid_tokens],
157 - query[num_prompt_tokens:num_valid_tokens],
158 - key_cache,
159 - value_cache,
160 - input_metadata)
191 + query[num_prompt_tokens:num_valid_tokens], key_cache,
192 + value_cache, input_metadata)
193
194 # Reshape the output tensor.
195 # NOTE(woosuk): The output tensor may include paddings.
@@ -179,9 +211,9 @@ class PagedAttentionWithRoPE(PagedAttention):
211 super().__init__(num_heads, head_size, scale)
212
213 # Create the cos and sin cache.
182 - inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
214 + inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
215 t = torch.arange(max_position).float()
184 - freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
216 + freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
217 cos = freqs.cos()
218 sin = freqs.sin()
219 cache = torch.cat((cos, sin), dim=-1)
@@ -195,15 +227,32 @@ class PagedAttentionWithRoPE(PagedAttention):
227
228 def forward(
229 self,
198 - positions: torch.Tensor, # [num_tokens]
199 - query: torch.Tensor, # [num_tokens, num_heads * head_size]
200 - key: torch.Tensor, # [num_tokens, num_heads * head_size]
201 - value: torch.Tensor, # [num_tokens, num_heads * head_size]
202 - key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
203 - value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
230 + positions: torch.Tensor,
231 + query: torch.Tensor,
232 + key: torch.Tensor,
233 + value: torch.Tensor,
234 + key_cache: torch.Tensor,
235 + value_cache: torch.Tensor,
236 input_metadata: InputMetadata,
237 cache_event: Optional[torch.cuda.Event],
206 - ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
238 + ) -> torch.Tensor:
239 + """ PagedAttention forward pass with rotary embedding.
240 +
241 + Args:
242 + positions: shape = [num_tokens]
243 + query: shape = [num_tokens, num_heads * head_size]
244 + key: shape = [num_tokens, num_heads * head_size]
245 + value: shape = [num_tokens, num_heads * head_size]
246 + key_cache: shape = [num_blocks, num_heads, head_size/x,
247 + block_size, x]
248 + value_cache: shape = [num_blocks, num_heads, head_size, block_size]
249 + input_metadata: metadata for paged attention.
250 + cache_event: event to wait for the cache operations to finish.
251 +
252 + Returns:
253 + shape = [num_tokens, num_heads * head_size]
254 + """
255 +
256 # Apply rotary embedding to the query and key before passing them
257 # to the attention op.
258 pos_encoding_ops.rotary_embedding_neox(
vllm/model_executor/layers/sampler.py
+45 -37
@@ -13,6 +13,7 @@ from vllm.sequence import SequenceOutputs
13
14 _SAMPLING_EPS = 1e-5
15
16 +
17 class Sampler(nn.Module):
18 """Samples the next tokens from the model's outputs.
19
@@ -50,19 +51,20 @@ class Sampler(nn.Module):
51 # Apply presence and frequency penalties.
52 output_tokens = _get_output_tokens(input_metadata)
53 assert len(output_tokens) == logits.shape[0]
53 - presence_penalties, frequency_penalties = _get_penalties(input_metadata)
54 + presence_penalties, frequency_penalties = _get_penalties(
55 + input_metadata)
56 assert len(presence_penalties) == logits.shape[0]
57 assert len(frequency_penalties) == logits.shape[0]
56 - logits = _apply_penalties(
57 - logits, output_tokens, presence_penalties, frequency_penalties,
58 - self.vocab_size)
58 + logits = _apply_penalties(logits, output_tokens, presence_penalties,
59 + frequency_penalties, self.vocab_size)
60
61 # Apply temperature scaling.
62 temperatures = _get_temperatures(input_metadata)
63 assert len(temperatures) == logits.shape[0]
64 if any(t != 1.0 for t in temperatures):
64 - t = torch.tensor(
65 - temperatures, dtype=logits.dtype, device=logits.device)
65 + t = torch.tensor(temperatures,
66 + dtype=logits.dtype,
67 + device=logits.device)
68 # Use in-place division to avoid creating a new tensor.
69 logits.div_(t.unsqueeze(dim=1))
70
@@ -75,7 +77,9 @@ class Sampler(nn.Module):
77 # Apply top-p and top-k truncation.
78 top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
79 assert len(top_ps) == len(top_ks) == probs.shape[0]
78 - if any(p < 1.0 - _SAMPLING_EPS for p in top_ps) or any(k != self.vocab_size for k in top_ks):
80 + do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
81 + do_top_k = any(k != self.vocab_size for k in top_ks)
82 + if do_top_p or do_top_k:
83 probs = _apply_top_p_top_k(probs, top_ps, top_ks)
84
85 # Sample the next tokens.
@@ -97,8 +101,7 @@ def _prune_hidden_states(
101
102
103 def _get_penalties(
100 - input_metadata: InputMetadata,
101 -) -> Tuple[List[float], List[float]]:
104 + input_metadata: InputMetadata) -> Tuple[List[float], List[float]]:
105 # Collect the presence and frequency penalties.
106 presence_penalties: List[float] = []
107 frequency_penalties: List[float] = []
@@ -117,9 +120,7 @@ def _get_penalties(
120 return presence_penalties, frequency_penalties
121
122
120 -def _get_output_tokens(
121 - input_metadata: InputMetadata,
122 -) -> List[List[int]]:
123 +def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
124 output_tokens: List[List[int]] = []
125 for i, seq_group in enumerate(input_metadata.seq_groups):
126 seq_ids, _ = seq_group
@@ -169,11 +170,13 @@ def _apply_penalties(
170 device=logits.device)
171
172 frequency_penalties = [frequency_penalties[i] for i in indices]
172 - frequency_penalties = torch.tensor(
173 - frequency_penalties, dtype=logits.dtype, device=logits.device)
173 + frequency_penalties = torch.tensor(frequency_penalties,
174 + dtype=logits.dtype,
175 + device=logits.device)
176 presence_penalties = [presence_penalties[i] for i in indices]
175 - presence_penalties = torch.tensor(
176 - presence_penalties, dtype=logits.dtype, device=logits.device)
177 + presence_penalties = torch.tensor(presence_penalties,
178 + dtype=logits.dtype,
179 + device=logits.device)
180
181 # We follow the definition in OpenAI API.
182 # Refer to https://platform.openai.com/docs/api-reference/parameter-details
@@ -183,9 +186,7 @@ def _apply_penalties(
186 return logits
187
188
186 -def _get_temperatures(
187 - input_metadata: InputMetadata,
188 -) -> List[float]:
189 +def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
190 # Collect the temperatures for the logits.
191 temperatures: List[float] = []
192 for i, seq_group in enumerate(input_metadata.seq_groups):
@@ -252,8 +253,9 @@ def _apply_top_p_top_k(
253 probs_sort[top_k_mask] = 0.0
254
255 # Re-sort the probabilities.
255 - probs = torch.gather(
256 - probs_sort, dim=-1, index=torch.argsort(probs_idx, dim=-1))
256 + probs = torch.gather(probs_sort,
257 + dim=-1,
258 + index=torch.argsort(probs_idx, dim=-1))
259 return probs
260
261
@@ -296,8 +298,9 @@ def _sample_from_prompt(
298 # Random sampling.
299 # Sample `best_of` tokens for the prompt.
300 num_seqs = sampling_params.best_of
299 - next_token_ids = torch.multinomial(
300 - prob, num_samples=num_seqs, replacement=True)
301 + next_token_ids = torch.multinomial(prob,
302 + num_samples=num_seqs,
303 + replacement=True)
304 next_token_ids = next_token_ids.tolist()
305 return next_token_ids
306
@@ -315,8 +318,9 @@ def _sample_from_generation_tokens(
318 if sampling_params.use_beam_search:
319 # Beam search.
320 # Add cumulative logprobs for the sequences in the group.
318 - seq_logprobs = torch.tensor(
319 - seq_logprobs, dtype=torch.float, device=logprobs.device)
321 + seq_logprobs = torch.tensor(seq_logprobs,
322 + dtype=torch.float,
323 + device=logprobs.device)
324 logprobs = logprobs + seq_logprobs.unsqueeze(dim=1)
325
326 vocab_size = logprobs.size(-1)
@@ -353,8 +357,9 @@ def _sample_from_generation_tokens(
357 else:
358 # Random sampling.
359 # Sample 1 token for each sequence in the group.
356 - next_token_ids = torch.multinomial(
357 - probs, num_samples=1, replacement=True)
360 + next_token_ids = torch.multinomial(probs,
361 + num_samples=1,
362 + replacement=True)
363 next_token_ids = next_token_ids.squeeze(dim=-1).tolist()
364 parent_seq_ids = seq_ids
365 return parent_seq_ids, next_token_ids
@@ -381,15 +386,16 @@ def _sample(
386 # Sample the next tokens.
387 next_token_ids = _sample_from_prompt(prob, sampling_params)
388 # Get top-k log probabilities for the next tokens.
384 - next_logprobs = _get_topk_logprobs(
385 - logprob, sampling_params.logprobs)
389 + next_logprobs = _get_topk_logprobs(logprob,
390 + sampling_params.logprobs)
391
392 # Build the output.
393 for seq_id, next_token_id in zip(seq_ids, next_token_ids):
394 output_logprobs = next_logprobs.copy()
395 output_logprobs[next_token_id] = logprob[next_token_id].item()
391 - seq_outputs[seq_id] = SequenceOutputs(
392 - seq_id, seq_id, next_token_id, output_logprobs)
396 + seq_outputs[seq_id] = SequenceOutputs(seq_id, seq_id,
397 + next_token_id,
398 + output_logprobs)
399 else:
400 # Generate the next tokens for generation tokens.
401 prob = probs[idx:idx + len(seq_ids)]
@@ -399,22 +405,24 @@ def _sample(
405 # Sample the next tokens.
406 seq_logprobs = [
407 input_metadata.seq_data[seq_id].cumulative_logprob
402 - for seq_id in seq_ids]
408 + for seq_id in seq_ids
409 + ]
410 parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
411 seq_ids, prob, logprob, seq_logprobs, sampling_params)
412
413 # Get top-k log probabilities for the next tokens.
414 next_logprobs: Dict[int, Dict[int, float]] = {}
408 - for i, seq_id in enumerate(seq_ids):
415 + for j, seq_id in enumerate(seq_ids):
416 next_logprobs[seq_id] = _get_topk_logprobs(
410 - logprob[i], sampling_params.logprobs)
417 + logprob[j], sampling_params.logprobs)
418
419 # Build the output.
420 for seq_id, parent_seq_id, next_token_id in zip(
414 - seq_ids, parent_seq_ids, next_token_ids):
415 - i = seq_ids.index(parent_seq_id)
421 + seq_ids, parent_seq_ids, next_token_ids):
422 + j = seq_ids.index(parent_seq_id)
423 output_logprobs = next_logprobs[parent_seq_id].copy()
417 - output_logprobs[next_token_id] = logprob[i, next_token_id].item()
424 + output_logprobs[next_token_id] = logprob[j,
425 + next_token_id].item()
426 seq_outputs[seq_id] = SequenceOutputs(
427 seq_id,
428 parent_seq_id,
vllm/model_executor/model_loader.py
+6 -7
@@ -6,8 +6,9 @@ import torch.nn as nn
6 from transformers import PretrainedConfig
7
8 from vllm.config import ModelConfig
9 -from vllm.model_executor.models import (GPT2LMHeadModel, GPTBigCodeForCausalLM, GPTNeoXForCausalLM,
10 - LlamaForCausalLM, OPTForCausalLM)
9 +from vllm.model_executor.models import (GPT2LMHeadModel, GPTBigCodeForCausalLM,
10 + GPTNeoXForCausalLM, LlamaForCausalLM,
11 + OPTForCausalLM)
12 from vllm.model_executor.weight_utils import initialize_dummy_weights
13
14 # TODO(woosuk): Lazy-load the model classes.
@@ -28,8 +29,7 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
29 return _MODEL_REGISTRY[arch]
30 raise ValueError(
31 f"Model architectures {architectures} are not supported for now. "
31 - f"Supported architectures: {list(_MODEL_REGISTRY.keys())}"
32 - )
32 + f"Supported architectures: {list(_MODEL_REGISTRY.keys())}")
33
34
35 def get_model(model_config: ModelConfig) -> nn.Module:
@@ -46,8 +46,7 @@ def get_model(model_config: ModelConfig) -> nn.Module:
46 initialize_dummy_weights(model)
47 else:
48 # Load the weights from the cached or downloaded files.
49 - model.load_weights(
50 - model_config.model, model_config.download_dir,
51 - model_config.use_np_weights)
49 + model.load_weights(model_config.model, model_config.download_dir,
50 + model_config.use_np_weights)
51 model = model.cuda()
52 return model.eval()
vllm/model_executor/models/__init__.py
-2
@@ -4,8 +4,6 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
4 from vllm.model_executor.models.llama import LlamaForCausalLM
5 from vllm.model_executor.models.opt import OPTForCausalLM
6
7 -
8 -
7 __all__ = [
8 "GPT2LMHeadModel",
9 "GPTBigCodeForCausalLM",
vllm/model_executor/models/gpt2.py
+53 -33
@@ -1,5 +1,6 @@
1 # coding=utf-8
2 -# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
4 # Copyright 2023 The vLLM team.
5 # Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
6 # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
@@ -47,19 +48,25 @@ class GPT2Attention(nn.Module):
48 super().__init__()
49 self.hidden_size = config.hidden_size
50 total_num_heads = config.num_attention_heads
50 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
51 + tensor_model_parallel_world_size = (
52 + get_tensor_model_parallel_world_size())
53 assert total_num_heads % tensor_model_parallel_world_size == 0
54 self.num_heads = total_num_heads // tensor_model_parallel_world_size
55 self.head_dim = self.hidden_size // total_num_heads
54 - self.scale = self.head_dim ** -0.5
56 + self.scale = self.head_dim**-0.5
57
56 - self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size,
57 - bias=True, gather_output=False,
58 + self.c_attn = ColumnParallelLinear(self.hidden_size,
59 + 3 * self.hidden_size,
60 + bias=True,
61 + gather_output=False,
62 perform_initialization=False)
59 - self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size,
60 - bias=True, input_is_parallel=True,
63 + self.c_proj = RowParallelLinear(self.hidden_size,
64 + self.hidden_size,
65 + bias=True,
66 + input_is_parallel=True,
67 perform_initialization=False)
62 - self.attn = PagedAttention(self.num_heads, self.head_dim,
68 + self.attn = PagedAttention(self.num_heads,
69 + self.head_dim,
70 scale=self.scale)
71
72 def forward(
@@ -72,8 +79,8 @@ class GPT2Attention(nn.Module):
79 qkv, _ = self.c_attn(hidden_states)
80 q, k, v = qkv.chunk(chunks=3, dim=-1)
81 key_cache, value_cache = kv_cache
75 - attn_output = self.attn(
76 - q, k, v, key_cache, value_cache, input_metadata, cache_event)
82 + attn_output = self.attn(q, k, v, key_cache, value_cache,
83 + input_metadata, cache_event)
84 attn_output, _ = self.c_proj(attn_output)
85 return attn_output
86
@@ -87,11 +94,15 @@ class GPT2MLP(nn.Module):
94 ):
95 super().__init__()
96 hidden_size = config.hidden_size
90 - self.c_fc = ColumnParallelLinear(hidden_size, intermediate_size,
91 - bias=True, gather_output=False,
97 + self.c_fc = ColumnParallelLinear(hidden_size,
98 + intermediate_size,
99 + bias=True,
100 + gather_output=False,
101 perform_initialization=False)
93 - self.c_proj = RowParallelLinear(intermediate_size, hidden_size,
94 - bias=True, input_is_parallel=True,
102 + self.c_proj = RowParallelLinear(intermediate_size,
103 + hidden_size,
104 + bias=True,
105 + input_is_parallel=True,
106 perform_initialization=False)
107 self.act = get_act_fn(config.activation_function)
108
@@ -107,7 +118,8 @@ class GPT2Block(nn.Module):
118 def __init__(self, config: GPT2Config):
119 super().__init__()
120 hidden_size = config.hidden_size
110 - inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
121 + inner_dim = (config.n_inner if config.n_inner is not None else 4 *
122 + hidden_size)
123
124 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
125 self.attn = GPT2Attention(config)
@@ -145,9 +157,9 @@ class GPT2Model(nn.Module):
157 def __init__(self, config: GPT2Config):
158 super().__init__()
159 self.config = config
148 - assert config.add_cross_attention == False
149 - assert config.scale_attn_by_inverse_layer_idx == False
150 - assert config.reorder_and_upcast_attn == False
160 + assert not config.add_cross_attention
161 + assert not config.scale_attn_by_inverse_layer_idx
162 + assert not config.reorder_and_upcast_attn
163 self.embed_dim = config.hidden_size
164
165 # Optimization: While the vocab size of GPT-2 is 50257, we extend it
@@ -180,8 +192,8 @@ class GPT2Model(nn.Module):
192 else:
193 cache_event = cache_events[i]
194 layer = self.h[i]
183 - hidden_states = layer(
184 - hidden_states, kv_caches[i], input_metadata, cache_event)
195 + hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
196 + cache_event)
197
198 hidden_states = self.ln_f(hidden_states)
199 return hidden_states
@@ -206,24 +218,26 @@ class GPT2LMHeadModel(nn.Module):
218 input_metadata: InputMetadata,
219 cache_events: Optional[List[torch.cuda.Event]],
220 ) -> Dict[int, SequenceOutputs]:
209 - hidden_states = self.transformer(
210 - input_ids, positions, kv_caches, input_metadata, cache_events)
211 - next_tokens = self.sampler(
212 - self.lm_head_weight, hidden_states, input_metadata)
221 + hidden_states = self.transformer(input_ids, positions, kv_caches,
222 + input_metadata, cache_events)
223 + next_tokens = self.sampler(self.lm_head_weight, hidden_states,
224 + input_metadata)
225 return next_tokens
226
227 _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
228 _row_parallel_weights = ["c_proj.weight"]
229
218 - def load_weights(self, model_name_or_path: str,
230 + def load_weights(self,
231 + model_name_or_path: str,
232 cache_dir: Optional[str] = None,
233 use_np_cache: bool = False):
221 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
234 + tensor_model_parallel_world_size = (
235 + get_tensor_model_parallel_world_size())
236 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
237 state_dict = self.state_dict()
238
239 for name, loaded_weight in hf_model_weights_iterator(
226 - model_name_or_path, cache_dir, use_np_cache):
240 + model_name_or_path, cache_dir, use_np_cache):
241 if "lm_head.weight" in name:
242 # GPT-2 ties the weights of the embedding layer and the final
243 # linear layer.
@@ -248,16 +262,20 @@ class GPT2LMHeadModel(nn.Module):
262
263 if name == "transformer.wte.weight":
264 # Consider padding in the vocab size.
251 - padded_vocab_size = param.shape[0] * tensor_model_parallel_world_size
265 + padded_vocab_size = (param.shape[0] *
266 + tensor_model_parallel_world_size)
267 num_extra_rows = padded_vocab_size - self.config.vocab_size
253 - extra_rows = torch.empty(num_extra_rows, loaded_weight.shape[1])
268 + extra_rows = torch.empty(num_extra_rows,
269 + loaded_weight.shape[1])
270 extra_rows = extra_rows.to(loaded_weight)
271 loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
272
273 # For the fused QKV linear layer, manually shard the weights.
274 if "c_attn" in name:
259 - # GPT-2's fused QKV has the shape of [3 * num_heads * head_size, hidden_size].
260 - # When tensor parallelism is used, we shard the weights along the head dimension.
275 + # GPT-2's fused QKV has the shape of
276 + # [3 * num_heads * head_size, hidden_size].
277 + # When tensor parallelism is used, we shard the weights along
278 + # the head dimension.
279 total_num_heads = self.config.num_attention_heads
280 hidden_size = self.config.hidden_size
281 head_size = hidden_size // total_num_heads
@@ -266,11 +284,13 @@ class GPT2LMHeadModel(nn.Module):
284 head_end = (tensor_model_parallel_rank + 1) * num_heads
285
286 if name.endswith(".weight"):
269 - loaded_weight = loaded_weight.view(3, total_num_heads, head_size, hidden_size)
287 + loaded_weight = loaded_weight.view(3, total_num_heads,
288 + head_size, hidden_size)
289 loaded_weight = loaded_weight[:, head_start:head_end, :, :]
290 loaded_weight = loaded_weight.reshape(-1, hidden_size)
291 elif name.endswith(".bias"):
273 - loaded_weight = loaded_weight.view(3, total_num_heads, head_size)
292 + loaded_weight = loaded_weight.view(3, total_num_heads,
293 + head_size)
294 loaded_weight = loaded_weight[:, head_start:head_end, :]
295 loaded_weight = loaded_weight.reshape(-1)
296 else:
vllm/model_executor/models/gpt_bigcode.py
+66 -38
@@ -1,5 +1,6 @@
1 # coding=utf-8
2 -# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
4 # Copyright 2023 The vLLM team.
5 # Copyright 2023 CTranslate2, and Michael Feil
6 # Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
@@ -49,19 +50,25 @@ class GPTBigCodeAttention(nn.Module):
50 super().__init__()
51 self.hidden_size = config.hidden_size
52 total_num_heads = config.num_attention_heads
52 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
53 + tensor_model_parallel_world_size = (
54 + get_tensor_model_parallel_world_size())
55 assert total_num_heads % tensor_model_parallel_world_size == 0
56 self.num_heads = total_num_heads // tensor_model_parallel_world_size
57 self.head_dim = self.hidden_size // total_num_heads
56 - self.scale = self.head_dim ** -0.5
58 + self.scale = self.head_dim**-0.5
59
58 - self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size,
59 - bias=True, gather_output=False,
60 + self.c_attn = ColumnParallelLinear(self.hidden_size,
61 + 3 * self.hidden_size,
62 + bias=True,
63 + gather_output=False,
64 perform_initialization=False)
61 - self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size,
62 - bias=True, input_is_parallel=True,
65 + self.c_proj = RowParallelLinear(self.hidden_size,
66 + self.hidden_size,
67 + bias=True,
68 + input_is_parallel=True,
69 perform_initialization=False)
64 - self.attn = PagedAttention(self.num_heads, self.head_dim,
70 + self.attn = PagedAttention(self.num_heads,
71 + self.head_dim,
72 scale=self.scale)
73
74 def forward(
@@ -74,8 +81,8 @@ class GPTBigCodeAttention(nn.Module):
81 qkv, _ = self.c_attn(hidden_states)
82 q, k, v = qkv.chunk(chunks=3, dim=-1)
83 key_cache, value_cache = kv_cache
77 - attn_output = self.attn(
78 - q, k, v, key_cache, value_cache, input_metadata, cache_event)
84 + attn_output = self.attn(q, k, v, key_cache, value_cache,
85 + input_metadata, cache_event)
86 attn_output, _ = self.c_proj(attn_output)
87 return attn_output
88
@@ -89,11 +96,15 @@ class GPTBigMLP(nn.Module):
96 ):
97 super().__init__()
98 hidden_size = config.hidden_size
92 - self.c_fc = ColumnParallelLinear(hidden_size, intermediate_size,
93 - bias=True, gather_output=False,
99 + self.c_fc = ColumnParallelLinear(hidden_size,
100 + intermediate_size,
101 + bias=True,
102 + gather_output=False,
103 perform_initialization=False)
95 - self.c_proj = RowParallelLinear(intermediate_size, hidden_size,
96 - bias=True, input_is_parallel=True,
104 + self.c_proj = RowParallelLinear(intermediate_size,
105 + hidden_size,
106 + bias=True,
107 + input_is_parallel=True,
108 perform_initialization=False)
109 self.act = get_act_fn(config.activation_function)
110
@@ -109,7 +120,8 @@ class GPTBigCodeBlock(nn.Module):
120 def __init__(self, config: GPTBigCodeConfig):
121 super().__init__()
122 hidden_size = config.hidden_size
112 - inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
123 + inner_dim = (config.n_inner if config.n_inner is not None else 4 *
124 + hidden_size)
125
126 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
127 self.attn = GPTBigCodeAttention(config)
@@ -147,7 +159,7 @@ class GPTBigCodeModel(nn.Module):
159 def __init__(self, config: GPTBigCodeConfig):
160 super().__init__()
161 self.config = config
150 - assert config.add_cross_attention == False
162 + assert not config.add_cross_attention
163
164 self.embed_dim = config.hidden_size
165
@@ -181,8 +193,8 @@ class GPTBigCodeModel(nn.Module):
193 else:
194 cache_event = cache_events[i]
195 layer = self.h[i]
184 - hidden_states = layer(
185 - hidden_states, kv_caches[i], input_metadata, cache_event)
196 + hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
197 + cache_event)
198
199 hidden_states = self.ln_f(hidden_states)
200 return hidden_states
@@ -207,24 +219,26 @@ class GPTBigCodeForCausalLM(nn.Module):
219 input_metadata: InputMetadata,
220 cache_events: Optional[List[torch.cuda.Event]],
221 ) -> Dict[int, SequenceOutputs]:
210 - hidden_states = self.transformer(
211 - input_ids, positions, kv_caches, input_metadata, cache_events)
212 - next_tokens = self.sampler(
213 - self.lm_head_weight, hidden_states, input_metadata)
222 + hidden_states = self.transformer(input_ids, positions, kv_caches,
223 + input_metadata, cache_events)
224 + next_tokens = self.sampler(self.lm_head_weight, hidden_states,
225 + input_metadata)
226 return next_tokens
227
228 _column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
229 _row_parallel_weights = ["c_proj.weight"]
230
219 - def load_weights(self, model_name_or_path: str,
231 + def load_weights(self,
232 + model_name_or_path: str,
233 cache_dir: Optional[str] = None,
234 use_np_cache: bool = False):
222 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
235 + tensor_model_parallel_world_size = (
236 + get_tensor_model_parallel_world_size())
237 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238 state_dict = self.state_dict()
239
240 for name, loaded_weight in hf_model_weights_iterator(
227 - model_name_or_path, cache_dir, use_np_cache):
241 + model_name_or_path, cache_dir, use_np_cache):
242 if "lm_head.weight" in name:
243 # GPT-2 ties the weights of the embedding layer and the final
244 # linear layer.
@@ -241,9 +255,11 @@ class GPTBigCodeForCausalLM(nn.Module):
255
256 if name == "transformer.wte.weight":
257 # Consider padding in the vocab size.
244 - padded_vocab_size = param.shape[0] * tensor_model_parallel_world_size
258 + padded_vocab_size = param.shape[
259 + 0] * tensor_model_parallel_world_size
260 num_extra_rows = padded_vocab_size - self.config.vocab_size
246 - extra_rows = torch.empty(num_extra_rows, loaded_weight.shape[1])
261 + extra_rows = torch.empty(num_extra_rows,
262 + loaded_weight.shape[1])
263 extra_rows = extra_rows.to(loaded_weight)
264 loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
265
@@ -258,25 +274,31 @@ class GPTBigCodeForCausalLM(nn.Module):
274 qkv_array = qkv_array.numpy()
275
276 dims_q = n_head * head_dim
261 - q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim), axis=0)
262 - # q is fine, but k & v have not replicated shape along the first axis
263 - # as long as MQA is not nativly supported, increase memory and replicated
264 - # (head_dim, hidden_dim) to (n_heads * head_dim, hidden_dim)
277 + # pylint: disable=unbalanced-tuple-unpacking
278 + q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim),
279 + axis=0)
280 + # q is fine, but k & v have not replicated shape along the first
281 + # axis as long as MQA is not nativly supported, increase memory
282 + # and replicated (head_dim, hidden_dim) to
283 + # (n_heads * head_dim, hidden_dim)
284 if k.ndim == 2 and v.ndim == 2:
285 replication = (n_head, 1) # weights
286 else:
287 replication = n_head # biases
288 # replicate n_head times for q, v
289 k, v = np.tile(k, replication), np.tile(v, replication)
271 - # concat q, k, v along the first axis (n_heads * head_dim, hidden_dim)
290 + # concat q, k, v along the first axis
291 + # (n_heads * head_dim, hidden_dim)
292 # to (3 * n_heads * head_dim, hidden_dim)
293 qkv_array = np.concatenate((q, k, v), axis=0)
294 return torch.from_numpy(qkv_array)
295
296 # For the fused QKV linear layer, manually shard the weights.
297 if "c_attn" in name:
278 - # GPT-2's fused QKV has the shape of [3 * num_heads * head_size, hidden_size].
279 - # When tensor parallelism is used, we shard the weights along the head dimension.
298 + # GPT-2's fused QKV has the shape of
299 + # [3 * num_heads * head_size, hidden_size].
300 + # When tensor parallelism is used, we shard the weights along
301 + # the head dimension.
302 total_num_heads = self.config.num_attention_heads
303 hidden_size = self.config.hidden_size
304 head_size = hidden_size // total_num_heads
@@ -285,13 +307,19 @@ class GPTBigCodeForCausalLM(nn.Module):
307 head_end = (tensor_model_parallel_rank + 1) * num_heads
308
309 if name.endswith(".weight"):
288 - loaded_weight = _expand_mqa_mha(loaded_weight, n_head=total_num_heads, head_dim=head_size)
289 - loaded_weight = loaded_weight.view(3, total_num_heads, head_size, hidden_size)
310 + loaded_weight = _expand_mqa_mha(loaded_weight,
311 + n_head=total_num_heads,
312 + head_dim=head_size)
313 + loaded_weight = loaded_weight.view(3, total_num_heads,
314 + head_size, hidden_size)
315 loaded_weight = loaded_weight[:, head_start:head_end, :, :]
316 loaded_weight = loaded_weight.reshape(-1, hidden_size)
317 elif name.endswith(".bias"):
293 - loaded_weight = _expand_mqa_mha(loaded_weight, n_head=total_num_heads, head_dim=head_size)
294 - loaded_weight = loaded_weight.view(3, total_num_heads, head_size)
318 + loaded_weight = _expand_mqa_mha(loaded_weight,
319 + n_head=total_num_heads,
320 + head_dim=head_size)
321 + loaded_weight = loaded_weight.view(3, total_num_heads,
322 + head_size)
323 loaded_weight = loaded_weight[:, head_start:head_end, :]
324 loaded_weight = loaded_weight.reshape(-1)
325 else:
vllm/model_executor/models/gpt_neox.py
+52 -33
@@ -1,5 +1,6 @@
1 # coding=utf-8
2 -# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
4 # Copyright 2023 The vLLM team.
5 # Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved.
6 #
@@ -48,19 +49,23 @@ class GPTNeoXAttention(nn.Module):
49 self.hidden_size = config.hidden_size
50 self.head_size = self.hidden_size // self.total_num_heads
51
51 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
52 + tensor_model_parallel_world_size = (
53 + get_tensor_model_parallel_world_size())
54 assert self.total_num_heads % tensor_model_parallel_world_size == 0
53 - self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
54 -
55 - self.query_key_value = ColumnParallelLinear(config.hidden_size,
56 - 3 * config.hidden_size,
57 - gather_output=False,
58 - perform_initialization=False)
59 - self.dense = RowParallelLinear(config.hidden_size, config.hidden_size,
55 + self.num_heads = (self.total_num_heads //
56 + tensor_model_parallel_world_size)
57 +
58 + self.query_key_value = ColumnParallelLinear(
59 + config.hidden_size,
60 + 3 * config.hidden_size,
61 + gather_output=False,
62 + perform_initialization=False)
63 + self.dense = RowParallelLinear(config.hidden_size,
64 + config.hidden_size,
65 input_is_parallel=True,
66 perform_initialization=False)
67
63 - scaling = self.head_size ** -0.5
68 + scaling = self.head_size**-0.5
69 rotary_dim = int(self.head_size * config.rotary_pct)
70 assert rotary_dim % 2 == 0
71 self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
@@ -78,8 +83,8 @@ class GPTNeoXAttention(nn.Module):
83
84 q, k, v = qkv.chunk(chunks=3, dim=-1)
85 k_cache, v_cache = kv_cache
81 - attn_output = self.attn(
82 - position_ids, q, k, v, k_cache, v_cache, input_metadata, cache_event)
86 + attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
87 + input_metadata, cache_event)
88 output, _ = self.dense(attn_output)
89 return output
90
@@ -92,7 +97,8 @@ class GPTNeoXMLP(nn.Module):
97 config.intermediate_size,
98 gather_output=False,
99 perform_initialization=False)
95 - self.dense_4h_to_h = RowParallelLinear(config.intermediate_size, config.hidden_size,
100 + self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
101 + config.hidden_size,
102 input_is_parallel=True,
103 perform_initialization=False)
104 self.act = get_act_fn(config.hidden_act)
@@ -109,8 +115,10 @@ class GPTNeoXLayer(nn.Module):
115 def __init__(self, config: GPTNeoXConfig):
116 super().__init__()
117 self.use_parallel_residual = config.use_parallel_residual
112 - self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
113 - self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
118 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
119 + eps=config.layer_norm_eps)
120 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
121 + eps=config.layer_norm_eps)
122 self.attention = GPTNeoXAttention(config)
123 self.mlp = GPTNeoXMLP(config)
124
@@ -154,10 +162,13 @@ class GPTNeoXModel(nn.Module):
162 super().__init__()
163 self.config = config
164
157 - self.embed_in = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
165 + self.embed_in = VocabParallelEmbedding(config.vocab_size,
166 + config.hidden_size,
167 perform_initialization=False)
159 - self.layers = nn.ModuleList([GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
160 - self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
168 + self.layers = nn.ModuleList(
169 + [GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
170 + self.final_layer_norm = nn.LayerNorm(config.hidden_size,
171 + eps=config.layer_norm_eps)
172
173 def forward(
174 self,
@@ -191,8 +202,10 @@ class GPTNeoXForCausalLM(nn.Module):
202 super().__init__()
203 self.config = config
204 self.gpt_neox = GPTNeoXModel(config)
194 - self.embed_out = ColumnParallelLinear(config.hidden_size, config.vocab_size,
195 - bias=False, gather_output=False,
205 + self.embed_out = ColumnParallelLinear(config.hidden_size,
206 + config.vocab_size,
207 + bias=False,
208 + gather_output=False,
209 perform_initialization=False)
210 self.sampler = Sampler(config.vocab_size)
211
@@ -204,24 +217,28 @@ class GPTNeoXForCausalLM(nn.Module):
217 input_metadata: InputMetadata,
218 cache_events: Optional[List[torch.cuda.Event]],
219 ) -> Dict[int, SequenceOutputs]:
207 - hidden_states = self.gpt_neox(
208 - input_ids, positions, kv_caches, input_metadata, cache_events)
209 - next_tokens = self.sampler(
210 - self.embed_out.weight, hidden_states, input_metadata)
220 + hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
221 + input_metadata, cache_events)
222 + next_tokens = self.sampler(self.embed_out.weight, hidden_states,
223 + input_metadata)
224 return next_tokens
225
213 - _column_parallel_weights = ["embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"]
226 + _column_parallel_weights = [
227 + "embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight",
228 + "dense_h_to_4h.bias"
229 + ]
230 _row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
231
216 - def load_weights(self, model_name_or_path: str,
232 + def load_weights(self,
233 + model_name_or_path: str,
234 cache_dir: Optional[str] = None,
235 use_np_cache: bool = False):
236 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
237 state_dict = self.state_dict()
238 for name, loaded_weight in hf_model_weights_iterator(
222 - model_name_or_path, cache_dir, use_np_cache):
239 + model_name_or_path, cache_dir, use_np_cache):
240 if ("attention.bias" in name or "attention.masked_bias" in name
224 - or "rotary_emb.inv_freq" in name):
241 + or "rotary_emb.inv_freq" in name):
242 continue
243 param = state_dict[name]
244 if "query_key_value" in name:
@@ -230,17 +247,19 @@ class GPTNeoXForCausalLM(nn.Module):
247 # required shape is [3 * num_heads * head_size, hidden_size].
248 # Thus, we need weight conversion.
249 shard_size = param.shape[0]
233 - loaded_weight = loaded_weight[shard_size * tensor_model_parallel_rank
234 - :shard_size * (tensor_model_parallel_rank + 1)]
250 + loaded_weight = loaded_weight[
251 + shard_size * tensor_model_parallel_rank:shard_size *
252 + (tensor_model_parallel_rank + 1)]
253
254 num_heads = self.config.num_attention_heads
255 hidden_size = self.config.hidden_size
256 head_size = hidden_size // num_heads
239 - if 'query_key_value.weight' in name:
240 - loaded_weight = loaded_weight.view(-1, 3, head_size, hidden_size)
257 + if "query_key_value.weight" in name:
258 + loaded_weight = loaded_weight.view(-1, 3, head_size,
259 + hidden_size)
260 loaded_weight = loaded_weight.transpose(0, 1)
261 loaded_weight = loaded_weight.reshape(-1, hidden_size)
243 - elif 'query_key_value.bias' in name:
262 + elif "query_key_value.bias" in name:
263 loaded_weight = loaded_weight.view(-1, 3, head_size)
264 loaded_weight = loaded_weight.transpose(0, 1)
265 loaded_weight = loaded_weight.reshape(-1)
vllm/model_executor/models/llama.py
+57 -39
@@ -1,5 +1,6 @@
1 # coding=utf-8
2 -# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 # Copyright 2023 The vLLM team.
5 # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 #
@@ -30,7 +31,6 @@ import torch
31 from torch import nn
32 from transformers import LlamaConfig
33
33 -from vllm.sequence import SequenceOutputs
34 from vllm.model_executor.input_metadata import InputMetadata
35 from vllm.model_executor.layers.activation import SiluAndMul
36 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -56,15 +56,19 @@ class LlamaMLP(nn.Module):
56 hidden_act: str,
57 ):
58 super().__init__()
59 - self.gate_up_proj = ColumnParallelLinear(hidden_size, 2 * intermediate_size,
60 - bias=False, gather_output=False,
59 + self.gate_up_proj = ColumnParallelLinear(hidden_size,
60 + 2 * intermediate_size,
61 + bias=False,
62 + gather_output=False,
63 perform_initialization=False)
62 - self.down_proj = RowParallelLinear(intermediate_size, hidden_size,
63 - bias=False, input_is_parallel=True,
64 + self.down_proj = RowParallelLinear(intermediate_size,
65 + hidden_size,
66 + bias=False,
67 + input_is_parallel=True,
68 perform_initialization=False)
65 - if hidden_act != 'silu':
66 - raise ValueError(f'Unsupported activation: {hidden_act}. '
67 - 'Only silu is supported for now.')
69 + if hidden_act != "silu":
70 + raise ValueError(f"Unsupported activation: {hidden_act}. "
71 + "Only silu is supported for now.")
72 self.act_fn = SiluAndMul()
73
74 def forward(self, x):
@@ -83,12 +87,14 @@ class LlamaAttention(nn.Module):
87 ):
88 super().__init__()
89 self.hidden_size = hidden_size
86 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
90 + tensor_model_parallel_world_size = (
91 + get_tensor_model_parallel_world_size())
92 self.total_num_heads = num_heads
93 assert self.total_num_heads % tensor_model_parallel_world_size == 0
89 - self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
94 + self.num_heads = (self.total_num_heads //
95 + tensor_model_parallel_world_size)
96 self.head_dim = hidden_size // self.total_num_heads
91 - self.scaling = self.head_dim ** -0.5
97 + self.scaling = self.head_dim**-0.5
98
99 self.qkv_proj = ColumnParallelLinear(
100 hidden_size,
@@ -104,8 +110,10 @@ class LlamaAttention(nn.Module):
110 input_is_parallel=True,
111 perform_initialization=False,
112 )
107 - self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_dim,
108 - self.scaling, rotary_dim=self.head_dim)
113 + self.attn = PagedAttentionWithRoPE(self.num_heads,
114 + self.head_dim,
115 + self.scaling,
116 + rotary_dim=self.head_dim)
117
118 def forward(
119 self,
@@ -118,8 +126,8 @@ class LlamaAttention(nn.Module):
126 qkv, _ = self.qkv_proj(hidden_states)
127 q, k, v = qkv.chunk(chunks=3, dim=-1)
128 k_cache, v_cache = kv_cache
121 - attn_output = self.attn(
122 - positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
129 + attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
130 + input_metadata, cache_event)
131 output, _ = self.o_proj(attn_output)
132 return output
133
@@ -138,8 +146,10 @@ class LlamaDecoderLayer(nn.Module):
146 intermediate_size=config.intermediate_size,
147 hidden_act=config.hidden_act,
148 )
141 - self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
142 - self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
149 + self.input_layernorm = RMSNorm(config.hidden_size,
150 + eps=config.rms_norm_eps)
151 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
152 + eps=config.rms_norm_eps)
153
154 def forward(
155 self,
@@ -177,9 +187,13 @@ class LlamaModel(nn.Module):
187 self.padding_idx = config.pad_token_id
188 self.vocab_size = config.vocab_size
189
180 - self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
181 - perform_initialization=False)
182 - self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
190 + self.embed_tokens = VocabParallelEmbedding(
191 + config.vocab_size,
192 + config.hidden_size,
193 + perform_initialization=False)
194 + self.layers = nn.ModuleList([
195 + LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)
196 + ])
197 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
198
199 def forward(
@@ -209,6 +223,7 @@ class LlamaModel(nn.Module):
223
224
225 class LlamaForCausalLM(nn.Module):
226 +
227 def __init__(self, config):
228 super().__init__()
229 self.config = config
@@ -228,39 +243,42 @@ class LlamaForCausalLM(nn.Module):
243 input_metadata: InputMetadata,
244 cache_events: Optional[List[torch.cuda.Event]],
245 ) -> Dict[int, SequenceOutputs]:
231 - hidden_states = self.model(
232 - input_ids, positions, kv_caches, input_metadata, cache_events)
233 - next_tokens = self.sampler(
234 - self.lm_head.weight, hidden_states, input_metadata)
246 + hidden_states = self.model(input_ids, positions, kv_caches,
247 + input_metadata, cache_events)
248 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
249 + input_metadata)
250 return next_tokens
251
237 - _column_parallel_weights = ["embed_tokens.weight", "lm_head.weight",
238 - "qkv_proj.weight", "gate_proj.weight",
239 - "up_proj.weight"]
252 + _column_parallel_weights = [
253 + "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
254 + "gate_proj.weight", "up_proj.weight"
255 + ]
256 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
257
242 - def load_weights(self, model_name_or_path: str,
258 + def load_weights(self,
259 + model_name_or_path: str,
260 cache_dir: Optional[str] = None,
261 use_np_cache: bool = False):
262 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
263 state_dict = self.state_dict()
264
265 for name, loaded_weight in hf_model_weights_iterator(
249 - model_name_or_path, cache_dir, use_np_cache):
266 + model_name_or_path, cache_dir, use_np_cache):
267 if "rotary_emb.inv_freq" in name:
268 continue
269
270 is_attention_weight = False
254 - for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
271 + for stride_id, att_weight_name in enumerate(
272 + ["q_proj", "k_proj", "v_proj"]):
273 if att_weight_name not in name:
274 continue
275 param = state_dict[name.replace(att_weight_name, "qkv_proj")]
276 shard_size = param.shape[0] // 3
277 loaded_weight = loaded_weight[
260 - shard_size * tensor_model_parallel_rank
261 - :shard_size * (tensor_model_parallel_rank + 1)]
262 - param_slice = param.data[shard_size * stride_id
263 - :shard_size * (stride_id + 1)]
278 + shard_size * tensor_model_parallel_rank:shard_size *
279 + (tensor_model_parallel_rank + 1)]
280 + param_slice = param.data[shard_size * stride_id:shard_size *
281 + (stride_id + 1)]
282 assert param_slice.shape == loaded_weight.shape
283 param_slice.copy_(loaded_weight)
284 is_attention_weight = True
@@ -275,10 +293,10 @@ class LlamaForCausalLM(nn.Module):
293 param = state_dict[name.replace(weight_name, "gate_up_proj")]
294 shard_size = param.shape[0] // 2
295 loaded_weight = loaded_weight[
278 - shard_size * tensor_model_parallel_rank
279 - :shard_size * (tensor_model_parallel_rank + 1)]
280 - param_slice = param.data[shard_size * stride_id
281 - :shard_size * (stride_id + 1)]
296 + shard_size * tensor_model_parallel_rank:shard_size *
297 + (tensor_model_parallel_rank + 1)]
298 + param_slice = param.data[shard_size * stride_id:shard_size *
299 + (stride_id + 1)]
300 assert param_slice.shape == loaded_weight.shape
301 param_slice.copy_(loaded_weight)
302 is_gate_up_weight = True
vllm/model_executor/models/opt.py
+69 -46
@@ -1,7 +1,9 @@
1 # coding=utf-8
2 -# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
4 # Copyright 2023 The vLLM team.
4 -# Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved.
5 +# Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights
6 +# reserved.
7 #
8 # Licensed under the Apache License, Version 2.0 (the "License");
9 # you may not use this file except in compliance with the License.
@@ -43,8 +45,9 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
45 class OPTLearnedPositionalEmbedding(nn.Embedding):
46
47 def __init__(self, num_embeddings: int, embedding_dim: int):
46 - # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2
47 - # and adjust num_embeddings appropriately. Other models don't have this hack
48 + # OPT is set up so that if padding_idx is specified then offset the
49 + # embedding ids by 2 and adjust num_embeddings appropriately. Other
50 + # models don't have this hack
51 self.offset = 2
52 super().__init__(num_embeddings + self.offset, embedding_dim)
53
@@ -62,20 +65,26 @@ class OPTAttention(nn.Module):
65 ) -> None:
66 super().__init__()
67 self.embed_dim = embed_dim
65 - tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
68 + tensor_model_parallel_world_size = (
69 + get_tensor_model_parallel_world_size())
70 total_num_heads = num_heads
71 assert num_heads % tensor_model_parallel_world_size == 0
72 self.num_heads = total_num_heads // tensor_model_parallel_world_size
73 self.head_dim = embed_dim // total_num_heads
70 - self.scaling = self.head_dim ** -0.5
74 + self.scaling = self.head_dim**-0.5
75
72 - self.qkv_proj = ColumnParallelLinear(embed_dim, 3 * embed_dim, bias=bias,
76 + self.qkv_proj = ColumnParallelLinear(embed_dim,
77 + 3 * embed_dim,
78 + bias=bias,
79 gather_output=False,
80 perform_initialization=False)
75 - self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
81 + self.out_proj = RowParallelLinear(embed_dim,
82 + embed_dim,
83 + bias=bias,
84 input_is_parallel=True,
85 perform_initialization=False)
78 - self.attn = PagedAttention(self.num_heads, self.head_dim,
86 + self.attn = PagedAttention(self.num_heads,
87 + self.head_dim,
88 scale=self.scaling)
89
90 def forward(
@@ -88,8 +97,8 @@ class OPTAttention(nn.Module):
97 qkv, _ = self.qkv_proj(hidden_states)
98 q, k, v = qkv.chunk(chunks=3, dim=-1)
99 key_cache, value_cache = kv_cache
91 - attn_output = self.attn(
92 - q, k, v, key_cache, value_cache, input_metadata, cache_event)
100 + attn_output = self.attn(q, k, v, key_cache, value_cache,
101 + input_metadata, cache_event)
102 output, _ = self.out_proj(attn_output)
103 return output
104
@@ -109,17 +118,21 @@ class OPTDecoderLayer(nn.Module):
118 self.activation_fn = get_act_fn(config.activation_function)
119
120 self.self_attn_layer_norm = nn.LayerNorm(
112 - self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
113 - self.fc1 = ColumnParallelLinear(self.embed_dim, config.ffn_dim,
121 + self.embed_dim,
122 + elementwise_affine=config.layer_norm_elementwise_affine)
123 + self.fc1 = ColumnParallelLinear(self.embed_dim,
124 + config.ffn_dim,
125 bias=config.enable_bias,
126 gather_output=False,
127 perform_initialization=False)
117 - self.fc2 = RowParallelLinear(config.ffn_dim, self.embed_dim,
128 + self.fc2 = RowParallelLinear(config.ffn_dim,
129 + self.embed_dim,
130 bias=config.enable_bias,
131 input_is_parallel=True,
132 perform_initialization=False)
133 self.final_layer_norm = nn.LayerNorm(
122 - self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
134 + self.embed_dim,
135 + elementwise_affine=config.layer_norm_elementwise_affine)
136
137 def forward(
138 self,
@@ -133,11 +146,10 @@ class OPTDecoderLayer(nn.Module):
146 # 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
147 if self.do_layer_norm_before:
148 hidden_states = self.self_attn_layer_norm(hidden_states)
136 - hidden_states = self.self_attn(
137 - hidden_states=hidden_states,
138 - kv_cache=kv_cache,
139 - input_metadata=input_metadata,
140 - cache_event=cache_event)
149 + hidden_states = self.self_attn(hidden_states=hidden_states,
150 + kv_cache=kv_cache,
151 + input_metadata=input_metadata,
152 + cache_event=cache_event)
153 hidden_states = residual + hidden_states
154 # 350m applies layer norm AFTER attention
155 if not self.do_layer_norm_before:
@@ -167,35 +179,42 @@ class OPTDecoder(nn.Module):
179 self.max_target_positions = config.max_position_embeddings
180 self.vocab_size = config.vocab_size
181
170 - self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
171 - config.word_embed_proj_dim,
172 - perform_initialization=False)
182 + self.embed_tokens = VocabParallelEmbedding(
183 + config.vocab_size,
184 + config.word_embed_proj_dim,
185 + perform_initialization=False)
186 # Positional embeddings are replicated (not sharded).
187 self.embed_positions = OPTLearnedPositionalEmbedding(
188 config.max_position_embeddings, config.hidden_size)
189
190 # Project out & in will be replicated if they exist.
191 if config.word_embed_proj_dim != config.hidden_size:
179 - self.project_out = nn.Linear(config.hidden_size, config.word_embed_proj_dim, bias=False)
192 + self.project_out = nn.Linear(config.hidden_size,
193 + config.word_embed_proj_dim,
194 + bias=False)
195 else:
196 self.project_out = None
197
198 if config.word_embed_proj_dim != config.hidden_size:
184 - self.project_in = nn.Linear(config.word_embed_proj_dim, config.hidden_size, bias=False)
199 + self.project_in = nn.Linear(config.word_embed_proj_dim,
200 + config.hidden_size,
201 + bias=False)
202 else:
203 self.project_in = None
204
188 - # Note that the only purpose of `config._remove_final_layer_norm` is to keep backward compatibility
189 - # with checkpoints that have been fine-tuned before transformers v4.20.1
205 + # Note that the only purpose of `config._remove_final_layer_norm` is to
206 + # keep backward compatibility with checkpoints that have been fine-tuned
207 + # before transformers v4.20.1
208 # see https://github.com/facebookresearch/metaseq/pull/164
209 if config.do_layer_norm_before and not config._remove_final_layer_norm:
210 self.final_layer_norm = nn.LayerNorm(
193 - config.hidden_size, elementwise_affine=config.layer_norm_elementwise_affine
194 - )
211 + config.hidden_size,
212 + elementwise_affine=config.layer_norm_elementwise_affine)
213 else:
214 self.final_layer_norm = None
215
198 - self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
216 + self.layers = nn.ModuleList(
217 + [OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
218
219 def forward(
220 self,
@@ -217,8 +236,8 @@ class OPTDecoder(nn.Module):
236 else:
237 cache_event = cache_events[i]
238 layer = self.layers[i]
220 - hidden_states = layer(
221 - hidden_states, kv_caches[i], input_metadata, cache_event)
239 + hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
240 + cache_event)
241
242 if self.final_layer_norm is not None:
243 hidden_states = self.final_layer_norm(hidden_states)
@@ -241,8 +260,8 @@ class OPTModel(nn.Module):
260 input_metadata: InputMetadata,
261 cache_events: Optional[List[torch.cuda.Event]],
262 ) -> torch.Tensor:
244 - return self.decoder(
245 - input_ids, positions, kv_caches, input_metadata, cache_events)
263 + return self.decoder(input_ids, positions, kv_caches, input_metadata,
264 + cache_events)
265
266
267 class OPTForCausalLM(nn.Module):
@@ -264,23 +283,26 @@ class OPTForCausalLM(nn.Module):
283 input_metadata: InputMetadata,
284 cache_events: Optional[List[torch.cuda.Event]],
285 ) -> Dict[int, SequenceOutputs]:
267 - hidden_states = self.model(
268 - input_ids, positions, kv_caches, input_metadata, cache_events)
269 - next_tokens = self.sampler(
270 - self.lm_head_weight, hidden_states, input_metadata)
286 + hidden_states = self.model(input_ids, positions, kv_caches,
287 + input_metadata, cache_events)
288 + next_tokens = self.sampler(self.lm_head_weight, hidden_states,
289 + input_metadata)
290 return next_tokens
291
273 - _column_parallel_weights = ["embed_tokens.weight", "fc1.weight", "fc1.bias"]
292 + _column_parallel_weights = [
293 + "embed_tokens.weight", "fc1.weight", "fc1.bias"
294 + ]
295 _row_parallel_weights = ["out_proj.weight", "fc2.weight"]
296
276 - def load_weights(self, model_name_or_path: str,
297 + def load_weights(self,
298 + model_name_or_path: str,
299 cache_dir: Optional[str] = None,
300 use_np_cache: bool = False):
301 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
302 state_dict = self.state_dict()
303
304 for name, loaded_weight in hf_model_weights_iterator(
283 - model_name_or_path, cache_dir, use_np_cache):
305 + model_name_or_path, cache_dir, use_np_cache):
306 if "lm_head.weight" in name:
307 continue
308
@@ -288,16 +310,17 @@ class OPTForCausalLM(nn.Module):
310 name = "model." + name
311
312 is_attention_weight = False
291 - for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
313 + for stride_id, att_weight_name in enumerate(
314 + ["q_proj", "k_proj", "v_proj"]):
315 if att_weight_name not in name:
316 continue
317 param = state_dict[name.replace(att_weight_name, "qkv_proj")]
318 shard_size = param.shape[0] // 3
319 loaded_weight = loaded_weight[
297 - shard_size * tensor_model_parallel_rank
298 - :shard_size * (tensor_model_parallel_rank + 1)]
299 - param_slice = param.data[shard_size * stride_id
300 - :shard_size * (stride_id + 1)]
320 + shard_size * tensor_model_parallel_rank:shard_size *
321 + (tensor_model_parallel_rank + 1)]
322 + param_slice = param.data[shard_size * stride_id:shard_size *
323 + (stride_id + 1)]
324 assert param_slice.shape == loaded_weight.shape
325 param_slice.copy_(loaded_weight)
326 is_attention_weight = True
vllm/model_executor/weight_utils.py
+10 -11
@@ -44,9 +44,9 @@ def hf_model_weights_iterator(
44 if use_np_cache:
45 # Convert the model weights from torch tensors to numpy arrays for
46 # faster loading.
47 - np_folder = os.path.join(hf_folder, 'np')
47 + np_folder = os.path.join(hf_folder, "np")
48 os.makedirs(np_folder, exist_ok=True)
49 - weight_names_file = os.path.join(np_folder, 'weight_names.json')
49 + weight_names_file = os.path.join(np_folder, "weight_names.json")
50 with lock:
51 if not os.path.exists(weight_names_file):
52 weight_names = []
@@ -57,10 +57,10 @@ def hf_model_weights_iterator(
57 with open(param_path, "wb") as f:
58 np.save(f, param.cpu().detach().numpy())
59 weight_names.append(name)
60 - with open(weight_names_file, 'w') as f:
60 + with open(weight_names_file, "w") as f:
61 json.dump(weight_names, f)
62
63 - with open(weight_names_file, 'r') as f:
63 + with open(weight_names_file, "r") as f:
64 weight_names = json.load(f)
65
66 for name in weight_names:
@@ -86,17 +86,16 @@ def load_tensor_parallel_weights(
86 for p in column_parallel_weight_names:
87 if p in param_name:
88 shard_size = param.shape[0]
89 - loaded_weight = loaded_weight[
90 - shard_size * tensor_model_parallel_rank
91 - :shard_size * (tensor_model_parallel_rank + 1)]
89 + start_idx = tensor_model_parallel_rank * shard_size
90 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
91 + loaded_weight = loaded_weight[start_idx:end_idx]
92 break
93 for p in row_parallel_weight_names:
94 if p in param_name:
95 shard_size = param.shape[1]
96 - loaded_weight = loaded_weight[
97 - :,
98 - shard_size * tensor_model_parallel_rank
99 - :shard_size * (tensor_model_parallel_rank + 1)]
96 + start_idx = tensor_model_parallel_rank * shard_size
97 + end_idx = (tensor_model_parallel_rank + 1) * shard_size
98 + loaded_weight = loaded_weight[:, start_idx:end_idx]
99 break
100 assert param.shape == loaded_weight.shape, (
101 f"{param_name} shape mismatch between model and checkpoint: "
vllm/outputs.py
+4 -2
@@ -55,6 +55,7 @@ class RequestOutput:
55 outputs: The output sequences of the request.
56 finished: Whether the whole request is finished.
57 """
58 +
59 def __init__(
60 self,
61 request_id: str,
@@ -75,8 +76,9 @@ class RequestOutput:
76 n = seq_group.sampling_params.n
77 seqs = seq_group.get_seqs()
78 assert n <= len(seqs)
78 - sorted_seqs = sorted(
79 - seqs, key=lambda seq: seq.get_cumulative_logprob(), reverse=True)
79 + sorted_seqs = sorted(seqs,
80 + key=lambda seq: seq.get_cumulative_logprob(),
81 + reverse=True)
82 top_n_seqs = sorted_seqs[:n]
83
84 # Create the outputs.
vllm/sampling_params.py
+8 -2
@@ -3,6 +3,7 @@ from typing import List, Optional, Union
3
4 _SAMPLING_EPS = 1e-5
5
6 +
7 class SamplingParams:
8 """Sampling parameters for text generation.
9
@@ -51,7 +52,7 @@ class SamplingParams:
52 top_p: float = 1.0,
53 top_k: int = -1,
54 use_beam_search: bool = False,
54 - stop: Union[str, List[str]] = [],
55 + stop: Union[None, str, List[str]] = None,
56 ignore_eos: bool = False,
57 max_tokens: int = 16,
58 logprobs: Optional[int] = None,
@@ -64,7 +65,12 @@ class SamplingParams:
65 self.top_p = top_p
66 self.top_k = top_k
67 self.use_beam_search = use_beam_search
67 - self.stop = [stop] if isinstance(stop, str) else list(stop)
68 + if stop is None:
69 + self.stop = []
70 + elif isinstance(stop, str):
71 + self.stop = [stop]
72 + else:
73 + self.stop = list(stop)
74 self.ignore_eos = ignore_eos
75 self.max_tokens = max_tokens
76 self.logprobs = logprobs
vllm/sequence.py
+70 -19
@@ -1,3 +1,4 @@
1 +"""Sequence and its related classes."""
2 import copy
3 import enum
4 from typing import Dict, List, Optional, Union
@@ -7,6 +8,7 @@ from vllm.sampling_params import SamplingParams
8
9
10 class SequenceStatus(enum.Enum):
11 + """Status of a sequence."""
12 WAITING = enum.auto()
13 RUNNING = enum.auto()
14 SWAPPED = enum.auto()
@@ -21,7 +23,7 @@ class SequenceStatus(enum.Enum):
23 SequenceStatus.FINISHED_STOPPED,
24 SequenceStatus.FINISHED_LENGTH_CAPPED,
25 SequenceStatus.FINISHED_ABORTED,
24 - SequenceStatus.FINISHED_IGNORED
26 + SequenceStatus.FINISHED_IGNORED,
27 ]
28
29 @staticmethod
@@ -40,6 +42,17 @@ class SequenceStatus(enum.Enum):
42
43
44 class SequenceData:
45 + """Data associated with a sequence.
46 +
47 +
48 + Args:
49 + prompt_token_ids: The token IDs of the prompt.
50 +
51 + Attributes:
52 + prompt_token_ids: The token IDs of the prompt.
53 + output_token_ids: The token IDs of the output.
54 + cumulative_logprob: The cumulative log probability of the output.
55 + """
56
57 def __init__(
58 self,
@@ -75,6 +88,15 @@ class SequenceData:
88
89
90 class Sequence:
91 + """Stores the data, status, and block information of a sequence.
92 +
93 + Args:
94 + seq_id: The ID of the sequence.
95 + prompt: The prompt of the sequence.
96 + prompt_token_ids: The token IDs of the prompt.
97 + block_size: The block size of the sequence. Should be the same as the
98 + block size used by the block manager and cache engine.
99 + """
100
101 def __init__(
102 self,
@@ -149,19 +171,27 @@ class Sequence:
171 def is_finished(self) -> bool:
172 return SequenceStatus.is_finished(self.status)
173
152 - def fork(self, child_seq: 'Sequence') -> None:
153 - child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
174 + def fork(self, child_seq: "Sequence") -> None:
175 + child_seq.logical_token_blocks = copy.deepcopy(
176 + self.logical_token_blocks)
177 child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
178 child_seq.data = copy.deepcopy(self.data)
156 - return None
179
180 def __repr__(self) -> str:
159 - return (f'Sequence(seq_id={self.seq_id}, '
160 - f'status={self.status.name}, '
161 - f'num_blocks={len(self.logical_token_blocks)})')
181 + return (f"Sequence(seq_id={self.seq_id}, "
182 + f"status={self.status.name}, "
183 + f"num_blocks={len(self.logical_token_blocks)})")
184
185
186 class SequenceGroup:
187 + """A group of sequences that are generated from the same prompt.
188 +
189 + Args:
190 + request_id: The ID of the request.
191 + seqs: The list of sequences.
192 + sampling_params: The sampling parameters used to generate the outputs.
193 + arrival_time: The arrival time of the request.
194 + """
195
196 def __init__(
197 self,
@@ -191,7 +221,7 @@ class SequenceGroup:
221 for seq in self.seqs:
222 if seq.seq_id == seq_id:
223 return seq
194 - raise ValueError(f'Sequence {seq_id} not found.')
224 + raise ValueError(f"Sequence {seq_id} not found.")
225
226 def is_finished(self) -> bool:
227 return all(seq.is_finished() for seq in self.seqs)
@@ -203,14 +233,25 @@ class SequenceGroup:
233
234
235 class SequenceGroupMetadata:
236 + """Metadata for a sequence group. Used to create `InputMetadata`.
237 +
238 +
239 + Args:
240 + request_id: The ID of the request.
241 + is_prompt: Whether the request is at prompt stage.
242 + seq_data: The sequence data. (Seq id -> sequence data)
243 + sampling_params: The sampling parameters used to generate the outputs.
244 + block_tables: The block tables. (Seq id -> list of physical block
245 + numbers)
246 + """
247
248 def __init__(
249 self,
250 request_id: str,
251 is_prompt: bool,
211 - seq_data: Dict[int, SequenceData], # Seq id -> sequence data.
252 + seq_data: Dict[int, SequenceData],
253 sampling_params: SamplingParams,
213 - block_tables: Dict[int, List[int]], # Seq id -> list of physical block numbers.
254 + block_tables: Dict[int, List[int]],
255 ) -> None:
256 self.request_id = request_id
257 self.is_prompt = is_prompt
@@ -220,13 +261,23 @@ class SequenceGroupMetadata:
261
262
263 class SequenceOutputs:
264 + """The model output associated with a sequence.
265 +
266 + Args:
267 + seq_id: The ID of the sequence.
268 + parent_seq_id: The ID of the parent sequence (for forking in beam
269 + search).
270 + output_token: The output token ID.
271 + logprobs: The logprobs of the output token.
272 + (Token id -> logP(x_i+1 | x_0, ..., x_i))
273 + """
274
275 def __init__(
276 self,
277 seq_id: int,
278 parent_seq_id: int,
279 output_token: int,
229 - logprobs: Dict[int, float], # Token id -> logP(x_i+1 | x_0, ..., x_i).
280 + logprobs: Dict[int, float],
281 ) -> None:
282 self.seq_id = seq_id
283 self.parent_seq_id = parent_seq_id
@@ -234,15 +285,15 @@ class SequenceOutputs:
285 self.logprobs = logprobs
286
287 def __repr__(self) -> str:
237 - return (f'SequenceOutputs(seq_id={self.seq_id}, '
238 - f'parent_seq_id={self.parent_seq_id}, '
239 - f'output_token={self.output_token}), '
240 - f'logprobs={self.logprobs}')
288 + return (f"SequenceOutputs(seq_id={self.seq_id}, "
289 + f"parent_seq_id={self.parent_seq_id}, "
290 + f"output_token={self.output_token}), "
291 + f"logprobs={self.logprobs}")
292
293 def __eq__(self, other: object) -> bool:
294 if not isinstance(other, SequenceOutputs):
295 return NotImplemented
245 - return (self.seq_id == other.seq_id and
246 - self.parent_seq_id == other.parent_seq_id and
247 - self.output_token == other.output_token and
248 - self.logprobs == other.logprobs)
296 + return (self.seq_id == other.seq_id
297 + and self.parent_seq_id == other.parent_seq_id
298 + and self.output_token == other.output_token
299 + and self.logprobs == other.logprobs)
vllm/transformers_utils/tokenizer.py
+3 -2
@@ -13,8 +13,8 @@ _FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
13
14 def get_tokenizer(
15 tokenizer_name: str,
16 - tokenizer_mode: str = "auto",
16 *args,
17 + tokenizer_mode: str = "auto",
18 **kwargs,
19 ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
20 """Gets a tokenizer for the given model name via Huggingface."""
@@ -73,7 +73,8 @@ def detokenize_incrementally(
73 output_text = tokenizer.convert_tokens_to_string(output_tokens)
74 return new_token, output_text
75
76 - # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
76 + # Adapted from
77 + # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
78 # NOTE(woosuk): The following code is slow because it runs a for loop over
79 # the output_tokens. In Python, running a for loop over a list can be slow
80 # even when the loop body is very simple.
vllm/utils.py
+3 -2
@@ -17,9 +17,9 @@ class Counter:
17 self.counter = start
18
19 def __next__(self) -> int:
20 - id = self.counter
20 + i = self.counter
21 self.counter += 1
22 - return id
22 + return i
23
24 def reset(self) -> None:
25 self.counter = 0
@@ -38,6 +38,7 @@ def get_cpu_memory() -> int:
38 def random_uuid() -> str:
39 return str(uuid.uuid4().hex)
40
41 +
42 def in_wsl() -> bool:
43 # Reference: https://github.com/microsoft/WSL/issues/4071
44 return "microsoft" in " ".join(uname()).lower()
vllm/worker/cache_engine.py
+5 -6
@@ -93,8 +93,8 @@ class CacheEngine:
93 if not pin_memory:
94 # Pinning memory in WSL is not supported.
95 # https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
96 - logger.warn("Using 'pin_memory=False' as WSL is detected. "
97 - "This may slow down the performance.")
96 + logger.warning("Using 'pin_memory=False' as WSL is detected. "
97 + "This may slow down the performance.")
98 for _ in range(self.num_layers):
99 key_blocks = torch.empty(
100 size=(self.num_cpu_blocks, *key_block_shape),
@@ -120,11 +120,10 @@ class CacheEngine:
120 src_key_cache, src_value_cache = src[i]
121 dst_key_cache, dst_value_cache = dst[i]
122 # Copy the key blocks.
123 - cache_ops.swap_blocks(
124 - src_key_cache, dst_key_cache, src_to_dst)
123 + cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
124 # Copy the value blocks.
126 - cache_ops.swap_blocks(
127 - src_value_cache, dst_value_cache, src_to_dst)
125 + cache_ops.swap_blocks(src_value_cache, dst_value_cache,
126 + src_to_dst)
127 event = self.events[i]
128 event.record(stream=self.cache_stream)
129
vllm/worker/worker.py
+13 -10
@@ -73,8 +73,8 @@ class Worker:
73 # number of tokens equal to max_num_batched_tokens.
74
75 # Enable top-k sampling to reflect the accurate memory usage.
76 - sampling_params = SamplingParams(top_p=0.99,
77 - top_k=self.model.config.vocab_size - 1)
76 + vocab_size = self.model.config.vocab_size
77 + sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
78 max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
79 max_num_seqs = self.scheduler_config.max_num_seqs
80 seqs = []
@@ -91,7 +91,8 @@ class Worker:
91 )
92 seqs.append(seq)
93
94 - input_tokens, input_positions, input_metadata = self._prepare_inputs(seqs)
94 + input_tokens, input_positions, input_metadata = self._prepare_inputs(
95 + seqs)
96
97 # Execute the model.
98 num_layers = self.model_config.get_num_layers(self.parallel_config)
@@ -110,8 +111,9 @@ class Worker:
111 total_gpu_memory = get_gpu_memory()
112 cache_block_size = CacheEngine.get_cache_block_size(
113 block_size, self.model_config, self.parallel_config)
113 - num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
114 - - peak_memory) // cache_block_size)
114 + num_gpu_blocks = int(
115 + (total_gpu_memory * gpu_memory_utilization - peak_memory) //
116 + cache_block_size)
117 num_cpu_blocks = int(cpu_swap_space // cache_block_size)
118 num_gpu_blocks = max(num_gpu_blocks, 0)
119 num_cpu_blocks = max(num_cpu_blocks, 0)
@@ -125,8 +127,8 @@ class Worker:
127 def init_cache_engine(self, cache_config: CacheConfig) -> None:
128 self.cache_config = cache_config
129 self.block_size = cache_config.block_size
128 - self.cache_engine = CacheEngine(
129 - self.cache_config, self.model_config, self.parallel_config)
130 + self.cache_engine = CacheEngine(self.cache_config, self.model_config,
131 + self.parallel_config)
132 self.cache_events = self.cache_engine.events
133 self.gpu_cache = self.cache_engine.gpu_cache
134
@@ -202,8 +204,8 @@ class Worker:
204 generation_block_tables.append(block_table)
205
206 max_context_len = max(max_context_len, context_len)
205 - max_num_blocks_per_seq = max(
206 - max_num_blocks_per_seq, len(block_table))
207 + max_num_blocks_per_seq = max(max_num_blocks_per_seq,
208 + len(block_table))
209 context_lens.append(context_len)
210
211 block_number = block_table[position // self.block_size]
@@ -223,7 +225,8 @@ class Worker:
225 context_lens_tensor = torch.cuda.IntTensor(context_lens)
226 padded_block_tables = [
227 _pad_to_max(block_table, max_num_blocks_per_seq)
226 - for block_table in generation_block_tables]
228 + for block_table in generation_block_tables
229 + ]
230 block_tables_tensor = torch.cuda.IntTensor(padded_block_tables)
231
232 seq_data: Dict[int, SequenceData] = {}