A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Quality] Add code formatter and linter (#326)
Zhuohan Li committed
Jul 3, 2023 at 11:31 UTC
d6fa1be3a8ef71fa16f74afdc5d07d27cbf725b1
47 files changed
+1549
-619
.pylintrc
new
+434
@@ -0,0 +1,434 @@
1
+# This Pylint rcfile contains a best-effort configuration to uphold the
2
+# best-practices and style described in the Google Python style guide:
3
+# https://google.github.io/styleguide/pyguide.html
4
+#
5
+# Its canonical open-source location is:
6
+# https://google.github.io/styleguide/pylintrc
7
+
8
+[MASTER]
9
+
10
+# Files or directories to be skipped. They should be base names, not paths.
11
+ignore=docs,parallel_utils
12
+
13
+# Files or directories matching the regex patterns are skipped. The regex
14
+# matches against base names, not paths.
15
+ignore-patterns=
16
+
17
+# Pickle collected data for later comparisons.
18
+persistent=no
19
+
20
+# List of plugins (as comma separated values of python modules names) to load,
21
+# usually to register additional checkers.
22
+load-plugins=
23
+
24
+# Use multiple processes to speed up Pylint.
25
+jobs=4
26
+
27
+# Allow loading of arbitrary C extensions. Extensions are imported into the
28
+# active Python interpreter and may run arbitrary code.
29
+unsafe-load-any-extension=no
30
+
31
+
32
+[MESSAGES CONTROL]
33
+
34
+# Only show warnings with the listed confidence levels. Leave empty to show
35
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
36
+confidence=
37
+
38
+# Enable the message, report, category or checker with the given id(s). You can
39
+# either give multiple identifier separated by comma (,) or put this option
40
+# multiple time (only on the command line, not in the configuration file where
41
+# it should appear only once). See also the "--disable" option for examples.
42
+#enable=
43
+
44
+# Disable the message, report, category or checker with the given id(s). You
45
+# can either give multiple identifiers separated by comma (,) or put this
46
+# option multiple times (only on the command line, not in the configuration
47
+# file where it should appear only once).You can also use "--disable=all" to
48
+# disable everything first and then reenable specific checks. For example, if
49
+# you want to run only the similarities checker, you can use "--disable=all
50
+# --enable=similarities". If you want to run only the classes checker, but have
51
+# no Warning level messages displayed, use"--disable=all --enable=classes
52
+# --disable=W"
53
+disable=abstract-method,
54
+ apply-builtin,
55
+ arguments-differ,
56
+ attribute-defined-outside-init,
57
+ backtick,
58
+ bad-option-value,
59
+ basestring-builtin,
60
+ buffer-builtin,
61
+ c-extension-no-member,
62
+ consider-using-enumerate,
63
+ cmp-builtin,
64
+ cmp-method,
65
+ coerce-builtin,
66
+ coerce-method,
67
+ delslice-method,
68
+ div-method,
69
+ duplicate-code,
70
+ eq-without-hash,
71
+ execfile-builtin,
72
+ file-builtin,
73
+ filter-builtin-not-iterating,
74
+ fixme,
75
+ getslice-method,
76
+ global-statement,
77
+ hex-method,
78
+ idiv-method,
79
+ implicit-str-concat-in-sequence,
80
+ import-error,
81
+ import-self,
82
+ import-star-module-level,
83
+ inconsistent-return-statements,
84
+ input-builtin,
85
+ intern-builtin,
86
+ invalid-str-codec,
87
+ locally-disabled,
88
+ logging-fstring-interpolation, # added by vLLM
89
+ logging-not-lazy, # added by vLLM
90
+ long-builtin,
91
+ long-suffix,
92
+ map-builtin-not-iterating,
93
+ misplaced-comparison-constant,
94
+ missing-class-docstring, # TODO (vLLM): enable
95
+ missing-function-docstring,
96
+ missing-module-docstring, # TODO (vLLM): enable
97
+ metaclass-assignment,
98
+ next-method-called,
99
+ next-method-defined,
100
+ no-absolute-import,
101
+ no-else-break,
102
+ no-else-continue,
103
+ no-else-raise,
104
+ no-else-return,
105
+ no-init, # added
106
+ no-member,
107
+ no-name-in-module,
108
+ no-self-use,
109
+ nonzero-method,
110
+ oct-method,
111
+ old-division,
112
+ old-ne-operator,
113
+ old-octal-literal,
114
+ old-raise-syntax,
115
+ parameter-unpacking,
116
+ print-statement,
117
+ raising-string,
118
+ range-builtin-not-iterating,
119
+ raw_input-builtin,
120
+ rdiv-method,
121
+ reduce-builtin,
122
+ relative-import,
123
+ reload-builtin,
124
+ round-builtin,
125
+ setslice-method,
126
+ signature-differs,
127
+ standarderror-builtin,
128
+ suppressed-message,
129
+ sys-max-int,
130
+ too-few-public-methods,
131
+ too-many-ancestors,
132
+ too-many-arguments,
133
+ too-many-boolean-expressions,
134
+ too-many-branches,
135
+ too-many-instance-attributes,
136
+ too-many-locals,
137
+ too-many-nested-blocks,
138
+ too-many-public-methods,
139
+ too-many-return-statements,
140
+ too-many-statements,
141
+ trailing-newlines,
142
+ unichr-builtin,
143
+ unicode-builtin,
144
+ unnecessary-pass,
145
+ unpacking-in-except,
146
+ unspecified-encoding,
147
+ useless-else-on-loop,
148
+ useless-object-inheritance,
149
+ useless-suppression,
150
+ using-cmp-argument,
151
+ wrong-import-order,
152
+ xrange-builtin,
153
+ zip-builtin-not-iterating,
154
+
155
+
156
+[REPORTS]
157
+
158
+# Set the output format. Available formats are text, parseable, colorized, msvs
159
+# (visual studio) and html. You can also give a reporter class, eg
160
+# mypackage.mymodule.MyReporterClass.
161
+output-format=text
162
+
163
+# Tells whether to display a full report or only the messages
164
+reports=no
165
+
166
+# Python expression which should return a note less than 10 (10 is the highest
167
+# note). You have access to the variables errors warning, statement which
168
+# respectively contain the number of errors / warnings messages and the total
169
+# number of statements analyzed. This is used by the global evaluation report
170
+# (RP0004).
171
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
172
+
173
+# Template used to display messages. This is a python new-style format string
174
+# used to format the message information. See doc for all details
175
+#msg-template=
176
+
177
+
178
+[BASIC]
179
+
180
+# Good variable names which should always be accepted, separated by a comma
181
+good-names=main,_
182
+
183
+# Bad variable names which should always be refused, separated by a comma
184
+bad-names=
185
+
186
+# Colon-delimited sets of names that determine each other's naming style when
187
+# the name regexes allow several styles.
188
+name-group=
189
+
190
+# Include a hint for the correct naming format with invalid-name
191
+include-naming-hint=no
192
+
193
+# List of decorators that produce properties, such as abc.abstractproperty. Add
194
+# to this list to register other decorators that produce valid properties.
195
+property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl
196
+
197
+# Regular expression matching correct function names
198
+function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
199
+
200
+# Regular expression matching correct variable names
201
+variable-rgx=^[a-z][a-z0-9_]*$
202
+
203
+# Regular expression matching correct constant names
204
+const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
205
+
206
+# Regular expression matching correct attribute names
207
+attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
208
+
209
+# Regular expression matching correct argument names
210
+argument-rgx=^[a-z][a-z0-9_]*$
211
+
212
+# Regular expression matching correct class attribute names
213
+class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
214
+
215
+# Regular expression matching correct inline iteration names
216
+inlinevar-rgx=^[a-z][a-z0-9_]*$
217
+
218
+# Regular expression matching correct class names
219
+class-rgx=^_?[A-Z][a-zA-Z0-9]*$
220
+
221
+# Regular expression matching correct module names
222
+module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$
223
+
224
+# Regular expression matching correct method names
225
+method-rgx=(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
226
+
227
+# Regular expression which should only match function or class names that do
228
+# not require a docstring.
229
+no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$
230
+
231
+# Minimum line length for functions/classes that require docstrings, shorter
232
+# ones are exempt.
233
+docstring-min-length=10
234
+
235
+
236
+[TYPECHECK]
237
+
238
+# List of decorators that produce context managers, such as
239
+# contextlib.contextmanager. Add to this list to register other decorators that
240
+# produce valid context managers.
241
+contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager
242
+
243
+# Tells whether missing members accessed in mixin class should be ignored. A
244
+# mixin class is detected if its name ends with "mixin" (case insensitive).
245
+ignore-mixin-members=yes
246
+
247
+# List of module names for which member attributes should not be checked
248
+# (useful for modules/projects where namespaces are manipulated during runtime
249
+# and thus existing member attributes cannot be deduced by static analysis. It
250
+# supports qualified module names, as well as Unix pattern matching.
251
+ignored-modules=
252
+
253
+# List of class names for which member attributes should not be checked (useful
254
+# for classes with dynamically set attributes). This supports the use of
255
+# qualified names.
256
+ignored-classes=optparse.Values,thread._local,_thread._local
257
+
258
+# List of members which are set dynamically and missed by pylint inference
259
+# system, and so shouldn't trigger E1101 when accessed. Python regular
260
+# expressions are accepted.
261
+generated-members=
262
+
263
+
264
+[FORMAT]
265
+
266
+# Maximum number of characters on a single line.
267
+max-line-length=80
268
+
269
+# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt
270
+# lines made too long by directives to pytype.
271
+
272
+# Regexp for a line that is allowed to be longer than the limit.
273
+ignore-long-lines=(?x)(
274
+ ^\s*(\#\ )?<?https?://\S+>?$|
275
+ ^\s*(from\s+\S+\s+)?import\s+.+$)
276
+
277
+# Allow the body of an if to be on the same line as the test if there is no
278
+# else.
279
+single-line-if-stmt=yes
280
+
281
+# Maximum number of lines in a module
282
+max-module-lines=99999
283
+
284
+# String used as indentation unit. The internal Google style guide mandates 2
285
+# spaces. Google's externaly-published style guide says 4, consistent with
286
+# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google
287
+# projects (like TensorFlow).
288
+indent-string=' '
289
+
290
+# Number of spaces of indent required inside a hanging or continued line.
291
+indent-after-paren=4
292
+
293
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
294
+expected-line-ending-format=
295
+
296
+
297
+[MISCELLANEOUS]
298
+
299
+# List of note tags to take in consideration, separated by a comma.
300
+notes=TODO
301
+
302
+
303
+[STRING]
304
+
305
+# This flag controls whether inconsistent-quotes generates a warning when the
306
+# character used as a quote delimiter is used inconsistently within a module.
307
+check-quote-consistency=yes
308
+
309
+
310
+[VARIABLES]
311
+
312
+# Tells whether we should check for unused import in __init__ files.
313
+init-import=no
314
+
315
+# A regular expression matching the name of dummy variables (i.e. expectedly
316
+# not used).
317
+dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_)
318
+
319
+# List of additional names supposed to be defined in builtins. Remember that
320
+# you should avoid to define new builtins when possible.
321
+additional-builtins=
322
+
323
+# List of strings which can identify a callback function by name. A callback
324
+# name must start or end with one of those strings.
325
+callbacks=cb_,_cb
326
+
327
+# List of qualified module names which can have objects that can redefine
328
+# builtins.
329
+redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools
330
+
331
+
332
+[LOGGING]
333
+
334
+# Logging modules to check that the string format arguments are in logging
335
+# function parameter format
336
+logging-modules=logging,absl.logging,tensorflow.io.logging
337
+
338
+
339
+[SIMILARITIES]
340
+
341
+# Minimum lines number of a similarity.
342
+min-similarity-lines=4
343
+
344
+# Ignore comments when computing similarities.
345
+ignore-comments=yes
346
+
347
+# Ignore docstrings when computing similarities.
348
+ignore-docstrings=yes
349
+
350
+# Ignore imports when computing similarities.
351
+ignore-imports=no
352
+
353
+
354
+[SPELLING]
355
+
356
+# Spelling dictionary name. Available dictionaries: none. To make it working
357
+# install python-enchant package.
358
+spelling-dict=
359
+
360
+# List of comma separated words that should not be checked.
361
+spelling-ignore-words=
362
+
363
+# A path to a file that contains private dictionary; one word per line.
364
+spelling-private-dict-file=
365
+
366
+# Tells whether to store unknown words to indicated private dictionary in
367
+# --spelling-private-dict-file option instead of raising a message.
368
+spelling-store-unknown-words=no
369
+
370
+
371
+[IMPORTS]
372
+
373
+# Deprecated modules which should not be used, separated by a comma
374
+deprecated-modules=regsub,
375
+ TERMIOS,
376
+ Bastion,
377
+ rexec,
378
+ sets
379
+
380
+# Create a graph of every (i.e. internal and external) dependencies in the
381
+# given file (report RP0402 must not be disabled)
382
+import-graph=
383
+
384
+# Create a graph of external dependencies in the given file (report RP0402 must
385
+# not be disabled)
386
+ext-import-graph=
387
+
388
+# Create a graph of internal dependencies in the given file (report RP0402 must
389
+# not be disabled)
390
+int-import-graph=
391
+
392
+# Force import order to recognize a module as part of the standard
393
+# compatibility libraries.
394
+known-standard-library=
395
+
396
+# Force import order to recognize a module as part of a third party library.
397
+known-third-party=enchant, absl
398
+
399
+# Analyse import fallback blocks. This can be used to support both Python 2 and
400
+# 3 compatible code, which means that the block might have code that exists
401
+# only in one or another interpreter, leading to false positives when analysed.
402
+analyse-fallback-blocks=no
403
+
404
+
405
+[CLASSES]
406
+
407
+# List of method names used to declare (i.e. assign) instance attributes.
408
+defining-attr-methods=__init__,
409
+ __new__,
410
+ setUp
411
+
412
+# List of member names, which should be excluded from the protected access
413
+# warning.
414
+exclude-protected=_asdict,
415
+ _fields,
416
+ _replace,
417
+ _source,
418
+ _make
419
+
420
+# List of valid names for the first argument in a class method.
421
+valid-classmethod-first-arg=cls,
422
+ class_
423
+
424
+# List of valid names for the first argument in a metaclass class method.
425
+valid-metaclass-classmethod-first-arg=mcs
426
+
427
+
428
+[EXCEPTIONS]
429
+
430
+# Exceptions that will emit a warning when being caught. Defaults to
431
+# "Exception"
432
+overgeneral-exceptions=StandardError,
433
+ Exception,
434
+ BaseException
CONTRIBUTING.md
+4
-1
@@ -49,12 +49,15 @@ If not, please file a new issue, providing as much relevant information as possi
49
50
In general, we adhere to [Google Python style guide](https://google.github.io/styleguide/pyguide.html) and [Google C++ style guide](https://google.github.io/styleguide/cppguide.html).
51
52
+We include a formatting script [`format.sh`](./format.sh) to format the code.
53
+
54
### Pull Requests
55
56
When submitting a pull request:
57
58
1. Make sure your code has been rebased on top of the latest commit on the main branch.
57
-2. Include a detailed description of the changes in the pull request.
59
+2. Ensure code is properly formatted by running [`format.sh`](./format.sh).
60
+3. Include a detailed description of the changes in the pull request.
61
Explain why you made the changes you did.
62
If your pull request fixes an open issue, please include a reference to it in the description.
63
examples/api_client.py
+5
-2
@@ -14,7 +14,9 @@ def clear_line(n: int = 1) -> None:
14
print(LINE_UP, end=LINE_CLEAR, flush=True)
15
16
17
-def post_http_request(prompt: str, api_url: str, n: int = 1,
17
+def post_http_request(prompt: str,
18
+ api_url: str,
19
+ n: int = 1,
20
stream: bool = False) -> requests.Response:
21
headers = {"User-Agent": "Test Client"}
22
pload = {
@@ -30,7 +32,8 @@ def post_http_request(prompt: str, api_url: str, n: int = 1,
32
33
34
def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:
33
- for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False,
35
+ for chunk in response.iter_lines(chunk_size=8192,
36
+ decode_unicode=False,
37
delimiter=b"\0"):
38
if chunk:
39
data = json.loads(chunk.decode("utf-8"))
examples/gradio_webserver.py
+16
-9
@@ -12,9 +12,14 @@ def http_bot(prompt):
12
"stream": True,
13
"max_tokens": 128,
14
}
15
- response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
16
-
17
- for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
15
+ response = requests.post(args.model_url,
16
+ headers=headers,
17
+ json=pload,
18
+ stream=True)
19
+
20
+ for chunk in response.iter_lines(chunk_size=8192,
21
+ decode_unicode=False,
22
+ delimiter=b"\0"):
23
if chunk:
24
data = json.loads(chunk.decode("utf-8"))
25
output = data["text"][0]
@@ -23,11 +28,11 @@ def http_bot(prompt):
28
29
def build_demo():
30
with gr.Blocks() as demo:
26
- gr.Markdown(
27
- "# vLLM text completion demo\n"
28
- )
29
- inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")
30
- outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
31
+ gr.Markdown("# vLLM text completion demo\n")
32
+ inputbox = gr.Textbox(label="Input",
33
+ placeholder="Enter text and press ENTER")
34
+ outputbox = gr.Textbox(label="Output",
35
+ placeholder="Generated result from the model")
36
inputbox.submit(http_bot, [inputbox], [outputbox])
37
return demo
38
@@ -36,7 +41,9 @@ if __name__ == "__main__":
41
parser = argparse.ArgumentParser()
42
parser.add_argument("--host", type=str, default="localhost")
43
parser.add_argument("--port", type=int, default=8001)
39
- parser.add_argument("--model-url", type=str, default="http://localhost:8000/generate")
44
+ parser.add_argument("--model-url",
45
+ type=str,
46
+ default="http://localhost:8000/generate")
47
args = parser.parse_args()
48
49
demo = build_demo()
examples/llm_engine_example.py
+7
-2
@@ -14,9 +14,14 @@ def main(args: argparse.Namespace):
14
("To be or not to be,",
15
SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
16
("What is the meaning of life?",
17
- SamplingParams(n=2, best_of=5, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
17
+ SamplingParams(n=2,
18
+ best_of=5,
19
+ temperature=0.8,
20
+ top_p=0.95,
21
+ frequency_penalty=0.1)),
22
("It is only with the heart that one can see rightly",
19
- SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
23
+ SamplingParams(n=3, best_of=3, use_beam_search=True,
24
+ temperature=0.0)),
25
]
26
27
# Run the engine by calling `engine.step()` manually.
examples/offline_inference.py
-1
@@ -1,6 +1,5 @@
1
from vllm import LLM, SamplingParams
2
3
-
3
# Sample prompts.
4
prompts = [
5
"Hello, my name is",
examples/openai_client.py
+7
-2
@@ -12,8 +12,13 @@ print("Models:", models)
12
# Test completion API
13
stream = True
14
completion = openai.Completion.create(
15
- model=model, prompt="A robot may not injure a human being", echo=False, n=2,
16
- best_of=3, stream=stream, logprobs=3)
15
+ model=model,
16
+ prompt="A robot may not injure a human being",
17
+ echo=False,
18
+ n=2,
19
+ best_of=3,
20
+ stream=stream,
21
+ logprobs=3)
22
23
# print the completion
24
if stream:
format.sh
new
+108
@@ -0,0 +1,108 @@
1
+#!/usr/bin/env bash
2
+# YAPF formatter, adapted from ray and skypilot.
3
+#
4
+# Usage:
5
+# # Do work and commit your work.
6
+
7
+# # Format files that differ from origin/main.
8
+# bash format.sh
9
+
10
+# # Commit changed files with message 'Run yapf and pylint'
11
+#
12
+#
13
+# YAPF + Clang formatter (if installed). This script formats all changed files from the last mergebase.
14
+# You are encouraged to run this locally before pushing changes for review.
15
+
16
+# Cause the script to exit if a single command fails
17
+set -eo pipefail
18
+
19
+# this stops git rev-parse from failing if we run this from the .git directory
20
+builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
21
+ROOT="$(git rev-parse --show-toplevel)"
22
+builtin cd "$ROOT" || exit 1
23
+
24
+YAPF_VERSION=$(yapf --version | awk '{print $2}')
25
+PYLINT_VERSION=$(pylint --version | head -n 1 | awk '{print $2}')
26
+MYPY_VERSION=$(mypy --version | awk '{print $2}')
27
+
28
+# # params: tool name, tool version, required version
29
+tool_version_check() {
30
+ if [[ $2 != $3 ]]; then
31
+ echo "Wrong $1 version installed: $3 is required, not $2."
32
+ exit 1
33
+ fi
34
+}
35
+
36
+tool_version_check "yapf" $YAPF_VERSION "$(grep yapf requirements-dev.txt | cut -d'=' -f3)"
37
+tool_version_check "pylint" $PYLINT_VERSION "$(grep "pylint==" requirements-dev.txt | cut -d'=' -f3)"
38
+tool_version_check "mypy" "$MYPY_VERSION" "$(grep mypy requirements-dev.txt | cut -d'=' -f3)"
39
+
40
+YAPF_FLAGS=(
41
+ '--recursive'
42
+ '--parallel'
43
+)
44
+
45
+YAPF_EXCLUDES=(
46
+ '--exclude' 'build/**'
47
+ '--exclude' 'vllm/model_executor/parallel_utils/**'
48
+)
49
+
50
+# Format specified files
51
+format() {
52
+ yapf --in-place "${YAPF_FLAGS[@]}" "$@"
53
+}
54
+
55
+# Format files that differ from main branch. Ignores dirs that are not slated
56
+# for autoformat yet.
57
+format_changed() {
58
+ # The `if` guard ensures that the list of filenames is not empty, which
59
+ # could cause yapf to receive 0 positional arguments, making it hang
60
+ # waiting for STDIN.
61
+ #
62
+ # `diff-filter=ACM` and $MERGEBASE is to ensure we only format files that
63
+ # exist on both branches.
64
+ MERGEBASE="$(git merge-base origin/main HEAD)"
65
+
66
+ if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' '*.pyi' &>/dev/null; then
67
+ git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' '*.pyi' | xargs -P 5 \
68
+ yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
69
+ fi
70
+
71
+}
72
+
73
+# Format all files
74
+format_all() {
75
+ yapf --in-place "${YAPF_FLAGS[@]}" "${YAPF_EXCLUDES[@]}" vllm
76
+}
77
+
78
+## This flag formats individual files. --files *must* be the first command line
79
+## arg to use this option.
80
+if [[ "$1" == '--files' ]]; then
81
+ format "${@:2}"
82
+ # If `--all` is passed, then any further arguments are ignored and the
83
+ # entire python directory is formatted.
84
+elif [[ "$1" == '--all' ]]; then
85
+ format_all
86
+else
87
+ # Format only the files that changed in last commit.
88
+ format_changed
89
+fi
90
+echo 'vLLM yapf: Done'
91
+
92
+# Run mypy
93
+# TODO(zhuohan): Enable mypy
94
+# echo 'vLLM mypy:'
95
+# mypy
96
+
97
+# Run Pylint
98
+echo 'vLLM Pylint:'
99
+pylint vllm
100
+
101
+if ! git diff --quiet &>/dev/null; then
102
+ echo 'Reformatted files. Please review and stage the changes.'
103
+ echo 'Changes not staged for commit:'
104
+ echo
105
+ git --no-pager diff --name-only
106
+
107
+ exit 1
108
+fi
requirements-dev.txt
+11
-1
@@ -1,2 +1,12 @@
1
-mypy
1
+# formatting
2
+yapf==0.32.0
3
+pylint==2.8.2
4
+
5
+# type checking
6
+mypy==0.991
7
+types-PyYAML
8
+types-requests
9
+types-setuptools
10
+
11
+# testing
12
pytest
tests/kernels/test_attention.py
+33
-20
@@ -60,7 +60,7 @@ def ref_single_query_cached_kv_attention(
60
keys = torch.stack(keys, dim=0)
61
values = torch.stack(values, dim=0)
62
63
- scale = 1.0 / (head_size ** 0.5)
63
+ scale = 1.0 / (head_size**0.5)
64
out = ref_masked_attention(q, keys, values, scale)
65
out = out.view(num_heads, head_size)
66
output[i].copy_(out, non_blocking=True)
@@ -74,7 +74,7 @@ def ref_multi_query_kv_attention(
74
dtype: torch.dtype,
75
) -> torch.Tensor:
76
head_size = query.shape[-1]
77
- scale = 1.0 / (head_size ** 0.5)
77
+ scale = 1.0 / (head_size**0.5)
78
79
num_seqs = len(cu_seq_lens) - 1
80
ref_outputs = []
@@ -84,8 +84,8 @@ def ref_multi_query_kv_attention(
84
seq_len = end_idx - start_idx
85
86
# Create attention mask.
87
- attn_mask = torch.triu(
88
- torch.ones(seq_len, seq_len, dtype=dtype), diagonal=1)
87
+ attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
88
+ diagonal=1)
89
attn_mask = attn_mask * torch.finfo(dtype).min
90
attn_mask = attn_mask.to(dtype=dtype, device='cuda')
91
@@ -113,7 +113,7 @@ def ref_multi_query_cached_kv_attention(
113
num_heads = value_cache.shape[1]
114
head_size = value_cache.shape[2]
115
block_size = value_cache.shape[3]
116
- scale = 1.0 / (head_size ** 0.5)
116
+ scale = 1.0 / (head_size**0.5)
117
118
num_queries = len(cu_query_lens) - 1
119
ref_outputs = []
@@ -125,8 +125,8 @@ def ref_multi_query_cached_kv_attention(
125
block_table = block_tables[i]
126
127
# Create attention mask
128
- attn_mask = torch.triu(
129
- torch.ones(query_len, context_len), diagonal=context_len - query_len + 1) * -1e5
128
+ attn_mask = torch.triu(torch.ones(query_len, context_len),
129
+ diagonal=context_len - query_len + 1) * -1e5
130
attn_mask = attn_mask.to(dtype=dtype, device='cuda')
131
132
keys = []
@@ -165,22 +165,28 @@ def run_single_query_cached_kv_attention(
165
num_blocks: int,
166
dtype: torch.dtype,
167
) -> None:
168
- qkv = torch.empty(
169
- num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
168
+ qkv = torch.empty(num_tokens,
169
+ 3,
170
+ num_heads,
171
+ head_size,
172
+ dtype=dtype,
173
+ device='cuda')
174
qkv.uniform_(-1e-3, 1e-3)
175
query, _, _ = qkv.unbind(dim=1)
176
177
x = 16 // torch.tensor([], dtype=dtype).element_size()
178
key_block_shape = (num_heads, head_size // x, block_size, x)
175
- key_cache = torch.empty(
176
- size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
179
+ key_cache = torch.empty(size=(num_blocks, *key_block_shape),
180
+ dtype=dtype,
181
+ device='cuda')
182
key_cache.uniform_(-1e-3, 1e-3)
183
value_block_shape = (num_heads, head_size, block_size)
179
- value_cache = torch.empty(
180
- size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
184
+ value_cache = torch.empty(size=(num_blocks, *value_block_shape),
185
+ dtype=dtype,
186
+ device='cuda')
187
value_cache.uniform_(-1e-3, 1e-3)
188
183
- context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
189
+ context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
190
max_context_len = max(context_lens)
191
context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
192
@@ -194,9 +200,12 @@ def run_single_query_cached_kv_attention(
200
block_tables.append(block_table)
201
block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
202
197
- scale = float(1.0 / (head_size ** 0.5))
198
- output = torch.empty(
199
- num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
203
+ scale = float(1.0 / (head_size**0.5))
204
+ output = torch.empty(num_tokens,
205
+ num_heads,
206
+ head_size,
207
+ dtype=dtype,
208
+ device='cuda')
209
attention_ops.single_query_cached_kv_attention(
210
output,
211
query,
@@ -235,9 +244,13 @@ def run_multi_query_kv_attention(
244
seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
245
num_tokens = sum(seq_lens)
246
238
- scale = float(1.0 / (head_size ** 0.5))
239
- qkv = torch.empty(
240
- num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
247
+ scale = float(1.0 / (head_size**0.5))
248
+ qkv = torch.empty(num_tokens,
249
+ 3,
250
+ num_heads,
251
+ head_size,
252
+ dtype=dtype,
253
+ device='cuda')
254
qkv.uniform_(-1e-3, 1e-3)
255
query, key, value = qkv.unbind(dim=1)
256
tests/kernels/test_cache.py
+61
-29
@@ -26,8 +26,9 @@ def run_copy_blocks(
26
key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
27
key_caches = []
28
for _ in range(num_layers):
29
- key_cache = torch.randn(
30
- size=key_cache_shape, dtype=dtype, device='cuda')
29
+ key_cache = torch.randn(size=key_cache_shape,
30
+ dtype=dtype,
31
+ device='cuda')
32
key_caches.append(key_cache)
33
cloned_key_caches = []
34
for key_cache in key_caches:
@@ -36,8 +37,9 @@ def run_copy_blocks(
37
value_cache_shape = (num_blocks, num_heads, head_size, block_size)
38
value_caches = []
39
for _ in range(num_layers):
39
- value_cache = torch.randn(
40
- size=value_cache_shape, dtype=dtype, device='cuda')
40
+ value_cache = torch.randn(size=value_cache_shape,
41
+ dtype=dtype,
42
+ device='cuda')
43
value_caches.append(value_cache)
44
cloned_value_caches = []
45
for value_cache in value_caches:
@@ -49,15 +51,18 @@ def run_copy_blocks(
51
# Reference implementation.
52
for src, dsts in block_mapping.items():
53
for dst in dsts:
52
- for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
54
+ for key_cache, cloned_key_cache in zip(key_caches,
55
+ cloned_key_caches):
56
cloned_key_cache[dst] = cloned_key_cache[src]
54
- for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
57
+ for value_cache, cloned_value_cache in zip(value_caches,
58
+ cloned_value_caches):
59
cloned_value_cache[dst] = cloned_value_cache[src]
60
61
# Compare the results.
62
for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
63
assert torch.allclose(key_cache, cloned_key_cache)
60
- for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
64
+ for value_cache, cloned_value_cache in zip(value_caches,
65
+ cloned_value_caches):
66
assert torch.allclose(value_cache, cloned_value_cache)
67
68
@@ -74,8 +79,12 @@ def run_reshape_and_cache(
79
slot_mapping = random.sample(range(num_slots), num_tokens)
80
slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
81
77
- qkv = torch.randn(
78
- num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
82
+ qkv = torch.randn(num_tokens,
83
+ 3,
84
+ num_heads,
85
+ head_size,
86
+ dtype=dtype,
87
+ device='cuda')
88
_, key, value = qkv.unbind(dim=1)
89
90
x = 16 // torch.tensor([], dtype=dtype).element_size()
@@ -84,15 +93,19 @@ def run_reshape_and_cache(
93
cloned_key_cache = key_cache.clone()
94
95
value_cache_shape = (num_blocks, num_heads, head_size, block_size)
87
- value_cache = torch.randn(
88
- size=value_cache_shape, dtype=dtype, device='cuda')
96
+ value_cache = torch.randn(size=value_cache_shape,
97
+ dtype=dtype,
98
+ device='cuda')
99
cloned_value_cache = value_cache.clone()
100
91
- cache_ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
101
+ cache_ops.reshape_and_cache(key, value, key_cache, value_cache,
102
+ slot_mapping)
103
104
for i in range(num_tokens):
105
reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
95
- block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
106
+ block_idx = torch.div(slot_mapping[i],
107
+ block_size,
108
+ rounding_mode='floor')
109
block_offset = slot_mapping[i] % block_size
110
cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
111
cloned_value_cache[block_idx, :, :, block_offset] = value[i]
@@ -114,8 +127,12 @@ def run_gather_cached_kv(
127
slot_mapping = random.sample(range(num_slots), num_tokens)
128
slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
129
117
- qkv = torch.randn(
118
- num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
130
+ qkv = torch.randn(num_tokens,
131
+ 3,
132
+ num_heads,
133
+ head_size,
134
+ dtype=dtype,
135
+ device='cuda')
136
_, key, value = qkv.unbind(dim=1)
137
138
qkv_clone = qkv.clone()
@@ -126,15 +143,20 @@ def run_gather_cached_kv(
143
key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
144
145
value_cache_shape = (num_blocks, num_heads, head_size, block_size)
129
- value_cache = torch.randn(
130
- size=value_cache_shape, dtype=dtype, device='cuda')
146
+ value_cache = torch.randn(size=value_cache_shape,
147
+ dtype=dtype,
148
+ device='cuda')
149
132
- cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
150
+ cache_ops.gather_cached_kv(key, value, key_cache, value_cache,
151
+ slot_mapping)
152
153
# Reference implementation.
154
for i in range(num_tokens):
136
- reshaped_key = cloned_key.reshape(num_tokens, num_heads, head_size // x, x)
137
- block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
155
+ reshaped_key = cloned_key.reshape(num_tokens, num_heads,
156
+ head_size // x, x)
157
+ block_idx = torch.div(slot_mapping[i],
158
+ block_size,
159
+ rounding_mode='floor')
160
block_offset = slot_mapping[i] % block_size
161
reshaped_key[i] = key_cache[block_idx, :, :, block_offset, :]
162
cloned_value[i] = value_cache[block_idx, :, :, block_offset]
@@ -145,20 +167,30 @@ def run_gather_cached_kv(
167
168
def test_copy_blocks() -> None:
169
for dtype in [torch.half, torch.bfloat16, torch.float]:
148
- run_copy_blocks(
149
- num_mappings=23, num_layers=7, num_heads=17, head_size=16,
150
- block_size=8, num_blocks=1024, dtype=dtype)
170
+ run_copy_blocks(num_mappings=23,
171
+ num_layers=7,
172
+ num_heads=17,
173
+ head_size=16,
174
+ block_size=8,
175
+ num_blocks=1024,
176
+ dtype=dtype)
177
178
179
def test_reshape_and_cache() -> None:
180
for dtype in [torch.half, torch.bfloat16, torch.float]:
155
- run_reshape_and_cache(
156
- num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
157
- dtype=dtype)
181
+ run_reshape_and_cache(num_tokens=3,
182
+ num_heads=2,
183
+ head_size=16,
184
+ block_size=8,
185
+ num_blocks=2,
186
+ dtype=dtype)
187
188
189
def test_gather_cached_kv() -> None:
190
for dtype in [torch.half, torch.bfloat16, torch.float]:
162
- run_gather_cached_kv(
163
- num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
164
- dtype=dtype)
191
+ run_gather_cached_kv(num_tokens=3,
192
+ num_heads=2,
193
+ head_size=16,
194
+ block_size=8,
195
+ num_blocks=2,
196
+ dtype=dtype)
tests/kernels/test_layernorm.py
+4
-2
@@ -14,8 +14,10 @@ class RefRMSNorm(nn.Module):
14
self.variance_epsilon = eps
15
16
def forward(self, hidden_states):
17
- variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
18
- hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
17
+ variance = hidden_states.to(torch.float32).pow(2).mean(-1,
18
+ keepdim=True)
19
+ hidden_states = hidden_states * torch.rsqrt(variance +
20
+ self.variance_epsilon)
21
if self.weight.dtype in [torch.half, torch.float16, torch.bfloat16]:
22
hidden_states = hidden_states.to(self.weight.dtype)
23
return self.weight * hidden_states
tests/kernels/test_pos_encoding.py
+20
-15
@@ -8,8 +8,8 @@ from vllm import pos_encoding_ops
8
9
10
def rotate_half(x: torch.Tensor) -> torch.Tensor:
11
- x1 = x[..., : x.shape[-1] // 2]
12
- x2 = x[..., x.shape[-1] // 2 :]
11
+ x1 = x[..., :x.shape[-1] // 2]
12
+ x2 = x[..., x.shape[-1] // 2:]
13
return torch.cat((-x2, x1), dim=-1)
14
15
@@ -38,7 +38,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
38
self.max_position_embeddings = max_position_embeddings
39
40
# Create cos and sin embeddings.
41
- inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2) / dim))
41
+ inv_freq = 1.0 / (base**(torch.arange(0, dim, 2) / dim))
42
t = torch.arange(max_position_embeddings).float()
43
freqs = torch.einsum("i,j->ij", t, inv_freq.float())
44
emb = torch.cat((freqs, freqs), dim=-1)
@@ -49,16 +49,15 @@ class RefRotaryEmbeddingNeox(nn.Module):
49
50
def forward(
51
self,
52
- positions: torch.Tensor, # [num_tokens]
53
- query: torch.Tensor, # [num_tokens, num_heads, head_size]
54
- key: torch.Tensor, # [num_tokens, num_heads, head_size]
52
+ positions: torch.Tensor, # [num_tokens]
53
+ query: torch.Tensor, # [num_tokens, num_heads, head_size]
54
+ key: torch.Tensor, # [num_tokens, num_heads, head_size]
55
) -> Tuple[torch.Tensor, torch.Tensor]:
56
57
- query_rot = query[..., : self.rotary_dim]
58
- query_pass = query[..., self.rotary_dim :]
59
- key_rot = key[..., : self.rotary_dim]
60
- key_pass = key[..., self.rotary_dim :]
61
-
57
+ query_rot = query[..., :self.rotary_dim]
58
+ query_pass = query[..., self.rotary_dim:]
59
+ key_rot = key[..., :self.rotary_dim]
60
+ key_pass = key[..., self.rotary_dim:]
61
62
query_rot = query_rot.transpose(0, 1)
63
key_rot = key_rot.transpose(0, 1)
@@ -85,12 +84,18 @@ def run_rotary_embedding_neox(
84
dtype: torch.dtype,
85
base: int = 10000,
86
) -> None:
88
- positions = torch.randint(0, max_position, (num_tokens,), device='cuda')
89
- query = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
90
- key = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
87
+ positions = torch.randint(0, max_position, (num_tokens, ), device='cuda')
88
+ query = torch.randn(num_tokens,
89
+ num_heads * head_size,
90
+ dtype=dtype,
91
+ device='cuda')
92
+ key = torch.randn(num_tokens,
93
+ num_heads * head_size,
94
+ dtype=dtype,
95
+ device='cuda')
96
97
# Create the rotary embedding.
93
- inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
98
+ inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
99
t = torch.arange(max_position).float()
100
freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
101
cos = freqs.cos()
vllm/__init__.py
+2
@@ -1,3 +1,5 @@
1
+"""vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
2
+
3
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4
from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
vllm/block.py
+2
-1
@@ -35,7 +35,8 @@ class LogicalTokenBlock:
35
36
def append_tokens(self, token_ids: List[int]) -> None:
37
assert len(token_ids) <= self.get_num_empty_slots()
38
- self.token_ids[self.num_tokens:self.num_tokens + len(token_ids)] = token_ids
38
+ curr_idx = self.num_tokens
39
+ self.token_ids[curr_idx:curr_idx + len(token_ids)] = token_ids
40
self.num_tokens += len(token_ids)
41
42
def get_token_ids(self) -> List[int]:
vllm/config.py
+12
-14
@@ -8,7 +8,7 @@ from vllm.utils import get_cpu_memory
8
9
logger = init_logger(__name__)
10
11
-_GiB = 1 << 30
11
+_GB = 1 << 30
12
13
14
class ModelConfig:
@@ -106,6 +106,7 @@ class CacheConfig:
106
vLLM execution.
107
swap_space: Size of the CPU swap space per GPU (in GiB).
108
"""
109
+
110
def __init__(
111
self,
112
block_size: int,
@@ -114,7 +115,7 @@ class CacheConfig:
115
) -> None:
116
self.block_size = block_size
117
self.gpu_memory_utilization = gpu_memory_utilization
117
- self.swap_space_bytes = swap_space * _GiB
118
+ self.swap_space_bytes = swap_space * _GB
119
self._verify_args()
120
121
# Will be set after profiling.
@@ -137,14 +138,13 @@ class CacheConfig:
138
num_gpus_per_node = parallel_config.tensor_parallel_size
139
cpu_memory_usage = self.swap_space_bytes * num_gpus_per_node
140
140
- msg = (
141
- f"{cpu_memory_usage / _GiB:.2f} GiB out of "
142
- f"the {total_cpu_memory / _GiB:.2f} GiB total CPU memory is "
143
- "allocated for the swap space.")
141
+ msg = (f"{cpu_memory_usage / _GB:.2f} GiB out of "
142
+ f"the {total_cpu_memory / _GB:.2f} GiB total CPU memory is "
143
+ "allocated for the swap space.")
144
if cpu_memory_usage > 0.7 * total_cpu_memory:
145
raise ValueError("Too large swap space. " + msg)
146
elif cpu_memory_usage > 0.4 * total_cpu_memory:
147
- logger.warn("Possibly too large swap space. " + msg)
147
+ logger.warning("Possibly too large swap space. " + msg)
148
149
150
class ParallelConfig:
@@ -157,6 +157,7 @@ class ParallelConfig:
157
True if either pipeline_parallel_size or tensor_parallel_size is
158
greater than 1.
159
"""
160
+
161
def __init__(
162
self,
163
pipeline_parallel_size: int,
@@ -189,12 +190,9 @@ class SchedulerConfig:
190
max_seq_len: Maximum length of a sequence (including prompt
191
and generated text).
192
"""
192
- def __init__(
193
- self,
194
- max_num_batched_tokens: int,
195
- max_num_seqs: int,
196
- max_seq_len: int
197
- ) -> None:
193
+
194
+ def __init__(self, max_num_batched_tokens: int, max_num_seqs: int,
195
+ max_seq_len: int) -> None:
196
self.max_num_batched_tokens = max_num_batched_tokens
197
self.max_num_seqs = max_num_seqs
198
self.max_seq_len = max_seq_len
@@ -241,7 +239,7 @@ def _get_and_verify_dtype(
239
pass
240
else:
241
# Casting between float16 and bfloat16 is allowed with a warning.
244
- logger.warn(f"Casting {config_dtype} to {torch_dtype}.")
242
+ logger.warning(f"Casting {config_dtype} to {torch_dtype}.")
243
244
# Check if the GPU supports the dtype.
245
if torch_dtype == torch.bfloat16:
vllm/core/block_manager.py
+9
-5
@@ -27,8 +27,9 @@ class BlockAllocator:
27
# Initialize the free blocks.
28
self.free_blocks: List[PhysicalTokenBlock] = []
29
for i in range(num_blocks):
30
- block = PhysicalTokenBlock(
31
- device=device, block_number=i, block_size=block_size)
30
+ block = PhysicalTokenBlock(device=device,
31
+ block_number=i,
32
+ block_size=block_size)
33
self.free_blocks.append(block)
34
35
def allocate(self) -> PhysicalTokenBlock:
@@ -84,10 +85,12 @@ class BlockSpaceManager:
85
num_required_blocks = len(seq.logical_token_blocks)
86
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
87
# Use watermark to avoid frequent cache eviction.
87
- return num_free_gpu_blocks - num_required_blocks >= self.watermark_blocks
88
+ return (num_free_gpu_blocks - num_required_blocks >=
89
+ self.watermark_blocks)
90
91
def allocate(self, seq_group: SequenceGroup) -> None:
90
- # NOTE: Here we assume that all sequences in the group have the same prompt.
92
+ # NOTE: Here we assume that all sequences in the group have the same
93
+ # prompt.
94
seq = seq_group.get_seqs()[0]
95
96
# Allocate new physical token blocks that will store the prompt tokens.
@@ -143,7 +146,8 @@ class BlockSpaceManager:
146
for block in src_block_table:
147
block.ref_count += 1
148
146
- def _get_physical_blocks(self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
149
+ def _get_physical_blocks(
150
+ self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
151
# NOTE: Here, we assume that the physical blocks are only shared by
152
# the sequences in the same group.
153
blocks: Set[PhysicalTokenBlock] = set()
vllm/core/scheduler.py
+35
-30
@@ -43,8 +43,7 @@ class SchedulerOutputs:
43
assert not (blocks_to_swap_in and blocks_to_swap_out)
44
45
def is_empty(self) -> bool:
46
- return (not self.blocks_to_swap_in
47
- and not self.blocks_to_swap_out
46
+ return (not self.blocks_to_swap_in and not self.blocks_to_swap_out
47
and not self.blocks_to_copy)
48
49
@@ -61,7 +60,7 @@ class Scheduler:
60
self.log_stats = log_stats
61
62
# Instantiate the scheduling policy.
64
- self.policy = PolicyFactory.get_policy(policy_name='fcfs')
63
+ self.policy = PolicyFactory.get_policy(policy_name="fcfs")
64
# Create the block space manager.
65
self.block_manager = BlockSpaceManager(
66
block_size=self.cache_config.block_size,
@@ -102,7 +101,8 @@ class Scheduler:
101
def get_num_unfinished_seq_groups(self) -> int:
102
return len(self.waiting) + len(self.running) + len(self.swapped)
103
105
- def _schedule(self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
104
+ def _schedule(
105
+ self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
106
# Blocks that need to be swaped or copied before model execution.
107
blocks_to_swap_in: Dict[int, int] = {}
108
blocks_to_swap_out: Dict[int, int] = {}
@@ -160,7 +160,8 @@ class Scheduler:
160
num_curr_seqs = sum(
161
seq_group.num_seqs(status=SequenceStatus.RUNNING)
162
for seq_group in self.running)
163
- if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
163
+ if (num_curr_seqs + num_new_seqs >
164
+ self.scheduler_config.max_num_seqs):
165
break
166
167
seq_group = self.swapped.pop(0)
@@ -170,8 +171,7 @@ class Scheduler:
171
172
num_batched_tokens = sum(
173
seq_group.num_seqs(status=SequenceStatus.RUNNING)
173
- for seq_group in self.running
174
- )
174
+ for seq_group in self.running)
175
176
# Join waiting sequences if possible.
177
prompt_group_ids: List[str] = []
@@ -191,7 +191,7 @@ class Scheduler:
191
192
num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193
if num_prompt_tokens >= self.scheduler_config.max_seq_len:
194
- logger.warn(
194
+ logger.warning(
195
f"Input prompt ({num_prompt_tokens} tokens) is too long"
196
" and exceeds limit of "
197
f"{self.scheduler_config.max_seq_len}")
@@ -206,17 +206,19 @@ class Scheduler:
206
break
207
208
# If the number of batched tokens exceeds the limit, stop.
209
- if (num_batched_tokens + num_prompt_tokens
210
- > self.scheduler_config.max_num_batched_tokens):
209
+ if (num_batched_tokens + num_prompt_tokens >
210
+ self.scheduler_config.max_num_batched_tokens):
211
break
212
213
# The total number of sequences in the RUNNING state should not
214
# exceed the maximum number of sequences.
215
- num_new_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
215
+ num_new_seqs = seq_group.num_seqs(
216
+ status=SequenceStatus.WAITING)
217
num_curr_seqs = sum(
218
seq_group.num_seqs(status=SequenceStatus.RUNNING)
219
for seq_group in self.running)
219
- if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
220
+ if (num_curr_seqs + num_new_seqs >
221
+ self.scheduler_config.max_num_seqs):
222
break
223
224
seq_group = self.waiting.pop(0)
@@ -240,12 +242,11 @@ class Scheduler:
242
elapsed_time = now - self.last_logging_time
243
if elapsed_time > _LOGGING_INTERVAL_SEC:
244
self.last_logging_time = now
243
- self.num_input_tokens = [
244
- (t, n) for t, n in self.num_input_tokens
245
- if now - t < _LOGGING_INTERVAL_SEC
246
- ]
245
+ self.num_input_tokens = [(t, n) for t, n in self.num_input_tokens
246
+ if now - t < _LOGGING_INTERVAL_SEC]
247
if len(self.num_input_tokens) > 1:
248
- total_num_tokens = sum(n for _, n in self.num_input_tokens[:-1])
248
+ total_num_tokens = sum(n
249
+ for _, n in self.num_input_tokens[:-1])
250
window = now - self.num_input_tokens[0][0]
251
avg_throughput = total_num_tokens / window
252
else:
@@ -258,26 +259,30 @@ class Scheduler:
259
260
total_num_cpu_blocks = self.cache_config.num_cpu_blocks
261
if total_num_cpu_blocks > 0:
261
- num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
262
+ num_free_cpu_blocks = (
263
+ self.block_manager.get_num_free_cpu_blocks())
264
num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
265
cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
266
else:
267
cpu_cache_usage = 0.0
268
267
- logger.info(
268
- f"Throughput: {avg_throughput:.1f} tokens/s, "
269
- f"Running: {len(self.running)} reqs, "
270
- f"Swapped: {len(self.swapped)} reqs, "
271
- f"Pending: {len(self.waiting)} reqs, "
272
- f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
273
- f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
269
+ logger.info(f"Throughput: {avg_throughput:.1f} tokens/s, "
270
+ f"Running: {len(self.running)} reqs, "
271
+ f"Swapped: {len(self.swapped)} reqs, "
272
+ f"Pending: {len(self.waiting)} reqs, "
273
+ f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
274
+ f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
275
return scheduler_outputs, prompt_group_ids, ignored_seq_groups
276
276
- def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs, List[SequenceGroup]]:
277
+ def schedule(
278
+ self
279
+ ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
280
+ List[SequenceGroup]]:
281
# Schedule sequence groups.
282
# This function call changes the internal states of the scheduler
283
# such as self.running, self.swapped, and self.waiting.
280
- scheduler_outputs, prompt_group_ids, ignored_seq_groups = self._schedule()
284
+ (scheduler_outputs, prompt_group_ids,
285
+ ignored_seq_groups) = self._schedule()
286
287
# Create input data structures.
288
seq_group_metadata_list: List[SequenceGroupMetadata] = []
@@ -311,8 +316,8 @@ class Scheduler:
316
for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
317
output = seq_outputs[seq.seq_id]
318
if seq.seq_id != output.parent_seq_id:
314
- # The sequence is a fork of the parent sequence (beam search).
315
- # Free the current sequence.
319
+ # The sequence is a fork of the parent sequence (beam
320
+ # search). Free the current sequence.
321
self.block_manager.free(seq)
322
# Fork the parent sequence.
323
parent_seq = seq_group.find(output.parent_seq_id)
@@ -385,7 +390,7 @@ class Scheduler:
390
elif preemption_mode == PreemptionMode.SWAP:
391
self._preempt_by_swap(seq_group, blocks_to_swap_out)
392
else:
388
- assert False, 'Invalid preemption mode.'
393
+ assert False, "Invalid preemption mode."
394
395
def _preempt_by_recompute(
396
self,
vllm/engine/arg_utils.py
+78
-51
@@ -12,11 +12,11 @@ class EngineArgs:
12
"""Arguments for vLLM engine."""
13
model: str
14
tokenizer: Optional[str] = None
15
- tokenizer_mode: str = "auto"
15
+ tokenizer_mode: str = 'auto'
16
download_dir: Optional[str] = None
17
use_np_weights: bool = False
18
use_dummy_weights: bool = False
19
- dtype: str = "auto"
19
+ dtype: str = 'auto'
20
seed: int = 0
21
worker_use_ray: bool = False
22
pipeline_parallel_size: int = 1
@@ -35,76 +35,101 @@ class EngineArgs:
35
36
@staticmethod
37
def add_cli_args(
38
- parser: argparse.ArgumentParser,
39
- ) -> argparse.ArgumentParser:
38
+ parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
39
"""Shared CLI arguments for vLLM engine."""
40
# Model arguments
42
- parser.add_argument('--model', type=str, default='facebook/opt-125m',
43
- help='name or path of the huggingface model to use')
44
- parser.add_argument('--tokenizer', type=str, default=EngineArgs.tokenizer,
45
- help='name or path of the huggingface tokenizer to use')
46
- parser.add_argument('--tokenizer-mode', type=str,
41
+ parser.add_argument(
42
+ '--model',
43
+ type=str,
44
+ default='facebook/opt-125m',
45
+ help='name or path of the huggingface model to use')
46
+ parser.add_argument(
47
+ '--tokenizer',
48
+ type=str,
49
+ default=EngineArgs.tokenizer,
50
+ help='name or path of the huggingface tokenizer to use')
51
+ parser.add_argument('--tokenizer-mode',
52
+ type=str,
53
default=EngineArgs.tokenizer_mode,
54
choices=['auto', 'slow'],
55
help='tokenizer mode. "auto" will use the fast '
50
- 'tokenizer if available, and "slow" will '
51
- 'always use the slow tokenizer.')
52
- parser.add_argument('--download-dir', type=str,
56
+ 'tokenizer if available, and "slow" will '
57
+ 'always use the slow tokenizer.')
58
+ parser.add_argument('--download-dir',
59
+ type=str,
60
default=EngineArgs.download_dir,
61
help='directory to download and load the weights, '
55
- 'default to the default cache dir of '
56
- 'huggingface')
57
- parser.add_argument('--use-np-weights', action='store_true',
62
+ 'default to the default cache dir of '
63
+ 'huggingface')
64
+ parser.add_argument('--use-np-weights',
65
+ action='store_true',
66
help='save a numpy copy of model weights for '
59
- 'faster loading. This can increase the disk '
60
- 'usage by up to 2x.')
61
- parser.add_argument('--use-dummy-weights', action='store_true',
67
+ 'faster loading. This can increase the disk '
68
+ 'usage by up to 2x.')
69
+ parser.add_argument('--use-dummy-weights',
70
+ action='store_true',
71
help='use dummy values for model weights')
72
# TODO(woosuk): Support FP32.
64
- parser.add_argument('--dtype', type=str, default=EngineArgs.dtype,
65
- choices=['auto', 'half', 'bfloat16', 'float'],
66
- help='data type for model weights and activations. '
67
- 'The "auto" option will use FP16 precision '
68
- 'for FP32 and FP16 models, and BF16 precision '
69
- 'for BF16 models.')
73
+ parser.add_argument(
74
+ '--dtype',
75
+ type=str,
76
+ default=EngineArgs.dtype,
77
+ choices=['auto', 'half', 'bfloat16', 'float'],
78
+ help='data type for model weights and activations. '
79
+ 'The "auto" option will use FP16 precision '
80
+ 'for FP32 and FP16 models, and BF16 precision '
81
+ 'for BF16 models.')
82
# Parallel arguments
71
- parser.add_argument('--worker-use-ray', action='store_true',
83
+ parser.add_argument('--worker-use-ray',
84
+ action='store_true',
85
help='use Ray for distributed serving, will be '
73
- 'automatically set when using more than 1 GPU')
74
- parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
86
+ 'automatically set when using more than 1 GPU')
87
+ parser.add_argument('--pipeline-parallel-size',
88
+ '-pp',
89
+ type=int,
90
default=EngineArgs.pipeline_parallel_size,
91
help='number of pipeline stages')
77
- parser.add_argument('--tensor-parallel-size', '-tp', type=int,
92
+ parser.add_argument('--tensor-parallel-size',
93
+ '-tp',
94
+ type=int,
95
default=EngineArgs.tensor_parallel_size,
96
help='number of tensor parallel replicas')
97
# KV cache arguments
81
- parser.add_argument('--block-size', type=int,
98
+ parser.add_argument('--block-size',
99
+ type=int,
100
default=EngineArgs.block_size,
101
choices=[8, 16, 32],
102
help='token block size')
103
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
86
- parser.add_argument('--seed', type=int, default=EngineArgs.seed,
104
+ parser.add_argument('--seed',
105
+ type=int,
106
+ default=EngineArgs.seed,
107
help='random seed')
88
- parser.add_argument('--swap-space', type=int,
108
+ parser.add_argument('--swap-space',
109
+ type=int,
110
default=EngineArgs.swap_space,
111
help='CPU swap space size (GiB) per GPU')
91
- parser.add_argument('--gpu-memory-utilization', type=float,
112
+ parser.add_argument('--gpu-memory-utilization',
113
+ type=float,
114
default=EngineArgs.gpu_memory_utilization,
115
help='the percentage of GPU memory to be used for'
94
- 'the model executor')
95
- parser.add_argument('--max-num-batched-tokens', type=int,
116
+ 'the model executor')
117
+ parser.add_argument('--max-num-batched-tokens',
118
+ type=int,
119
default=EngineArgs.max_num_batched_tokens,
120
help='maximum number of batched tokens per '
98
- 'iteration')
99
- parser.add_argument('--max-num-seqs', type=int,
121
+ 'iteration')
122
+ parser.add_argument('--max-num-seqs',
123
+ type=int,
124
default=EngineArgs.max_num_seqs,
125
help='maximum number of sequences per iteration')
102
- parser.add_argument('--disable-log-stats', action='store_true',
126
+ parser.add_argument('--disable-log-stats',
127
+ action='store_true',
128
help='disable logging statistics')
129
return parser
130
131
@classmethod
107
- def from_cli_args(cls, args: argparse.Namespace) -> "EngineArgs":
132
+ def from_cli_args(cls, args: argparse.Namespace) -> 'EngineArgs':
133
# Get the list of attributes of this dataclass.
134
attrs = [attr.name for attr in dataclasses.fields(cls)]
135
# Set the attributes from the parsed arguments.
@@ -115,18 +140,19 @@ class EngineArgs:
140
self,
141
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
142
# Initialize the configs.
118
- model_config = ModelConfig(
119
- self.model, self.tokenizer, self.tokenizer_mode, self.download_dir,
120
- self.use_np_weights, self.use_dummy_weights, self.dtype, self.seed)
121
- cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
143
+ model_config = ModelConfig(self.model, self.tokenizer,
144
+ self.tokenizer_mode, self.download_dir,
145
+ self.use_np_weights, self.use_dummy_weights,
146
+ self.dtype, self.seed)
147
+ cache_config = CacheConfig(self.block_size,
148
+ self.gpu_memory_utilization,
149
self.swap_space)
150
parallel_config = ParallelConfig(self.pipeline_parallel_size,
151
self.tensor_parallel_size,
152
self.worker_use_ray)
126
- max_seq_len = min(
127
- self.max_num_batched_tokens,
128
- getattr(model_config.hf_config, "max_position_embeddings",
129
- float("inf")))
153
+ model_max_len = getattr(model_config.hf_config,
154
+ 'max_position_embeddings', float('inf'))
155
+ max_seq_len = min(self.max_num_batched_tokens, model_max_len)
156
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
157
self.max_num_seqs, max_seq_len)
158
return model_config, cache_config, parallel_config, scheduler_config
@@ -140,12 +166,13 @@ class AsyncEngineArgs(EngineArgs):
166
167
@staticmethod
168
def add_cli_args(
143
- parser: argparse.ArgumentParser,
144
- ) -> argparse.ArgumentParser:
169
+ parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
170
parser = EngineArgs.add_cli_args(parser)
146
- parser.add_argument('--engine-use-ray', action='store_true',
171
+ parser.add_argument('--engine-use-ray',
172
+ action='store_true',
173
help='use Ray to start the LLM engine in a '
148
- 'separate process as the server process.')
149
- parser.add_argument('--disable-log-requests', action='store_true',
174
+ 'separate process as the server process.')
175
+ parser.add_argument('--disable-log-requests',
176
+ action='store_true',
177
help='disable logging requests')
178
return parser
vllm/engine/async_llm_engine.py
+26
-17
@@ -11,7 +11,7 @@ from vllm.sampling_params import SamplingParams
11
12
logger = init_logger(__name__)
13
14
-TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
14
+TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17
class AsyncLLMEngine:
@@ -35,8 +35,13 @@ class AsyncLLMEngine:
35
log_requests: Whether to log the requests.
36
*args, *kwargs: Arguments for LLMEngine.
37
"""
38
- def __init__(self, worker_use_ray: bool, engine_use_ray: bool,
39
- log_requests: bool = True, *args, **kwargs) -> None:
38
+
39
+ def __init__(self,
40
+ worker_use_ray: bool,
41
+ engine_use_ray: bool,
42
+ *args,
43
+ log_requests: bool = True,
44
+ **kwargs) -> None:
45
self.worker_use_ray = worker_use_ray
46
self.engine_use_ray = engine_use_ray
47
self.log_requests = log_requests
@@ -76,12 +81,11 @@ class AsyncLLMEngine:
81
self.request_events[request_id].set()
82
83
async def generate(
79
- self,
80
- prompt: Optional[str],
81
- sampling_params: SamplingParams,
82
- request_id: str,
83
- prompt_token_ids: Optional[List[int]] = None
84
- ) -> RequestOutput:
84
+ self,
85
+ prompt: Optional[str],
86
+ sampling_params: SamplingParams,
87
+ request_id: str,
88
+ prompt_token_ids: Optional[List[int]] = None) -> RequestOutput:
89
"""Generate outputs for a request.
90
91
Generate outputs for a request. This method is a coroutine. It adds the
@@ -117,14 +121,17 @@ class AsyncLLMEngine:
121
# Add the request into the vLLM engine's waiting queue.
122
if self.engine_use_ray:
123
await self.engine.add_request.remote(
120
- request_id, prompt, sampling_params,
124
+ request_id,
125
+ prompt,
126
+ sampling_params,
127
prompt_token_ids=prompt_token_ids,
128
arrival_time=arrival_time)
129
else:
124
- self.engine.add_request(
125
- request_id, prompt, sampling_params,
126
- prompt_token_ids=prompt_token_ids,
127
- arrival_time=arrival_time)
130
+ self.engine.add_request(request_id,
131
+ prompt,
132
+ sampling_params,
133
+ prompt_token_ids=prompt_token_ids,
134
+ arrival_time=arrival_time)
135
136
# The vLLM engine does not have a background loop that keeps
137
# processing incoming requests. Therefore, we need to keep kicking
@@ -200,7 +207,8 @@ class AsyncLLMEngine:
207
self.kicking_request_id = None
208
209
@classmethod
203
- def from_engine_args(cls, engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
210
+ def from_engine_args(cls,
211
+ engine_args: AsyncEngineArgs) -> "AsyncLLMEngine":
212
"""Creates an async LLM engine from the engine arguments."""
213
# Create the engine configs.
214
engine_configs = engine_args.create_engine_configs()
@@ -211,8 +219,9 @@ class AsyncLLMEngine:
219
# Create the async LLM engine.
220
engine = cls(engine_args.worker_use_ray,
221
engine_args.engine_use_ray,
214
- not engine_args.disable_log_requests,
222
*engine_configs,
216
- distributed_init_method, devices,
223
+ distributed_init_method,
224
+ devices,
225
+ log_requests=not engine_args.disable_log_requests,
226
log_stats=not engine_args.disable_log_stats)
227
return engine
vllm/engine/llm_engine.py
+18
-15
@@ -67,8 +67,7 @@ class LLMEngine:
67
f"download_dir={model_config.download_dir!r}, "
68
f"use_np_weights={model_config.use_np_weights}, "
69
f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
70
- f"seed={model_config.seed})"
71
- )
70
+ f"seed={model_config.seed})")
71
# TODO(woosuk): Print more configs in debug mode.
72
73
self.model_config = model_config
@@ -78,8 +77,8 @@ class LLMEngine:
77
self.log_stats = log_stats
78
self._verify_args()
79
81
- self.tokenizer = get_tokenizer(model_config.tokenizer,
82
- model_config.tokenizer_mode)
80
+ self.tokenizer = get_tokenizer(
81
+ model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
82
self.seq_counter = Counter()
83
84
# Create the parallel GPU workers.
@@ -129,8 +128,8 @@ class LLMEngine:
128
num_gpu_blocks = min(b[0] for b in num_blocks)
129
num_cpu_blocks = min(b[1] for b in num_blocks)
130
# FIXME(woosuk): Change to debug log.
132
- logger.info(f'# GPU blocks: {num_gpu_blocks}, '
133
- f'# CPU blocks: {num_cpu_blocks}')
131
+ logger.info(f"# GPU blocks: {num_gpu_blocks}, "
132
+ f"# CPU blocks: {num_cpu_blocks}")
133
134
if num_gpu_blocks <= 0:
135
raise ValueError("No available memory for the cache blocks. "
@@ -152,7 +151,9 @@ class LLMEngine:
151
# Initialize the cluster.
152
distributed_init_method, devices = initialize_cluster(parallel_config)
153
# Create the LLM engine.
155
- engine = cls(*engine_configs, distributed_init_method, devices,
154
+ engine = cls(*engine_configs,
155
+ distributed_init_method,
156
+ devices,
157
log_stats=not engine_args.disable_log_stats)
158
return engine
159
@@ -226,8 +227,10 @@ class LLMEngine:
227
and updates the scheduler with the model outputs. Finally, it decodes
228
the sequences and returns the newly generated results.
229
"""
229
- seq_group_metadata_list, scheduler_outputs, ignored_seq_groups = self.scheduler.schedule()
230
- if (not seq_group_metadata_list) and scheduler_outputs.is_empty() and (not ignored_seq_groups):
230
+ (seq_group_metadata_list, scheduler_outputs,
231
+ ignored_seq_groups) = self.scheduler.schedule()
232
+ if ((not seq_group_metadata_list) and scheduler_outputs.is_empty()
233
+ and (not ignored_seq_groups)):
234
# Nothing to do.
235
return []
236
@@ -281,8 +284,8 @@ class LLMEngine:
284
# Truncate the output text so that the stop string is
285
# not included in the output.
286
seq.output_text = seq.output_text[:-len(stop_str)]
284
- self.scheduler.free_seq(seq,
285
- SequenceStatus.FINISHED_STOPPED)
287
+ self.scheduler.free_seq(
288
+ seq, SequenceStatus.FINISHED_STOPPED)
289
stopped = True
290
break
291
if stopped:
@@ -290,7 +293,7 @@ class LLMEngine:
293
294
# Check if the sequence has reached max_seq_len.
295
if (seq.get_len() >=
293
- self.scheduler.scheduler_config.max_seq_len):
296
+ self.scheduler.scheduler_config.max_seq_len):
297
self.scheduler.free_seq(
298
seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
299
continue
@@ -302,15 +305,15 @@ class LLMEngine:
305
# Check if the sequence has generated the EOS token.
306
if not sampling_params.ignore_eos:
307
if seq.get_last_token_id() == self.tokenizer.eos_token_id:
305
- self.scheduler.free_seq(seq,
306
- SequenceStatus.FINISHED_STOPPED)
308
+ self.scheduler.free_seq(
309
+ seq, SequenceStatus.FINISHED_STOPPED)
310
continue
311
312
def _run_workers(
313
self,
314
method: str,
312
- get_all_outputs: bool = False,
315
*args,
316
+ get_all_outputs: bool = False,
317
**kwargs,
318
) -> Any:
319
"""Runs the given method on all workers."""
vllm/engine/ray_utils.py
+7
-6
@@ -8,7 +8,8 @@ except ImportError:
8
9
from vllm.config import ParallelConfig
10
11
-DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), device id
11
+# rank, node resource (node IP), device id
12
+DeviceID = Tuple[int, Optional[str], int]
13
14
15
def initialize_cluster(
@@ -53,15 +54,15 @@ def initialize_cluster(
54
valid_node_resources = []
55
num_devices_per_node = None
56
for node in ray.nodes():
56
- if (not node['Alive']) or node['Resources']['GPU'] <= 0:
57
+ if (not node["Alive"]) or node["Resources"]["GPU"] <= 0:
58
continue
59
if num_devices_per_node is None:
59
- num_devices_per_node = node['Resources']['GPU']
60
+ num_devices_per_node = node["Resources"]["GPU"]
61
else:
61
- assert num_devices_per_node == node['Resources']['GPU'], (
62
+ assert num_devices_per_node == node["Resources"]["GPU"], (
63
"The number of GPUs per node is not uniform.")
63
- for key in node['Resources']:
64
- if key.startswith('node:'):
64
+ for key in node["Resources"]:
65
+ if key.startswith("node:"):
66
valid_node_resources.append(key)
67
68
# Verify the parallel config.
vllm/entrypoints/api_server.py
+8
-9
@@ -11,8 +11,8 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.sampling_params import SamplingParams
12
from vllm.utils import random_uuid
13
14
-TIMEOUT_KEEP_ALIVE = 5 # seconds.
15
-TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
14
+TIMEOUT_KEEP_ALIVE = 5 # seconds.
15
+TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds.
16
app = FastAPI()
17
18
@@ -37,8 +37,7 @@ async def generate(request: Request) -> Response:
37
async for request_output in results_generator:
38
prompt = request_output.prompt
39
text_outputs = [
40
- prompt + output.text
41
- for output in request_output.outputs
40
+ prompt + output.text for output in request_output.outputs
41
]
42
ret = {"text": text_outputs}
43
yield (json.dumps(ret) + "\0").encode("utf-8")
@@ -63,10 +62,7 @@ async def generate(request: Request) -> Response:
62
63
assert final_output is not None
64
prompt = final_output.prompt
66
- text_outputs = [
67
- prompt + output.text
68
- for output in final_output.outputs
69
- ]
65
+ text_outputs = [prompt + output.text for output in final_output.outputs]
66
ret = {"text": text_outputs}
67
return Response(content=json.dumps(ret))
68
@@ -81,5 +77,8 @@ if __name__ == "__main__":
77
engine_args = AsyncEngineArgs.from_cli_args(args)
78
engine = AsyncLLMEngine.from_engine_args(engine_args)
79
84
- uvicorn.run(app, host=args.host, port=args.port, log_level="debug",
80
+ uvicorn.run(app,
81
+ host=args.host,
82
+ port=args.port,
83
+ log_level="debug",
84
timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
vllm/entrypoints/llm.py
+1
-2
@@ -63,8 +63,7 @@ class LLM:
63
self.request_counter = Counter()
64
65
def get_tokenizer(
66
- self,
67
- ) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
66
+ self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
67
return self.llm_engine.tokenizer
68
69
def set_tokenizer(
vllm/entrypoints/openai/api_server.py
+67
-46
@@ -1,4 +1,5 @@
1
-# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
1
+# Adapted from
2
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
3
4
import argparse
5
from http import HTTPStatus
@@ -29,7 +30,7 @@ from vllm.sampling_params import SamplingParams
30
from vllm.transformers_utils.tokenizer import get_tokenizer
31
from vllm.utils import random_uuid
32
32
-TIMEOUT_KEEP_ALIVE = 5 # seconds
33
+TIMEOUT_KEEP_ALIVE = 5 # seconds
34
35
logger = init_logger(__name__)
36
served_model = None
@@ -38,14 +39,13 @@ app = fastapi.FastAPI()
39
40
def create_error_response(status_code: HTTPStatus,
41
message: str) -> JSONResponse:
41
- return JSONResponse(
42
- ErrorResponse(message=message, type="invalid_request_error").dict(),
43
- status_code=status_code.value
44
- )
42
+ return JSONResponse(ErrorResponse(message=message,
43
+ type="invalid_request_error").dict(),
44
+ status_code=status_code.value)
45
46
47
@app.exception_handler(RequestValidationError)
48
-async def validation_exception_handler(request, exc):
48
+async def validation_exception_handler(request, exc): # pylint: disable=unused-argument
49
return create_error_response(HTTPStatus.BAD_REQUEST, str(exc))
50
51
@@ -126,8 +126,11 @@ async def check_length(request, prompt, engine):
126
@app.get("/v1/models")
127
async def show_available_models():
128
"""Show available models. Right now we only have one model."""
129
- model_cards = [ModelCard(id=served_model, root=served_model,
130
- permission=[ModelPermission()])]
129
+ model_cards = [
130
+ ModelCard(id=served_model,
131
+ root=served_model,
132
+ permission=[ModelPermission()])
133
+ ]
134
return ModelList(data=model_cards)
135
136
@@ -144,12 +147,14 @@ def create_logprobs(token_ids: List[int],
147
if len(logprobs.text_offset) == 0:
148
logprobs.text_offset.append(initial_text_offset)
149
else:
147
- logprobs.text_offset.append(logprobs.text_offset[-1] + last_token_len)
150
+ logprobs.text_offset.append(logprobs.text_offset[-1] +
151
+ last_token_len)
152
last_token_len = len(token)
153
150
- logprobs.top_logprobs.append(
151
- {tokenizer.convert_ids_to_tokens(i): p
152
- for i, p in id_logprob.items()})
154
+ logprobs.top_logprobs.append({
155
+ tokenizer.convert_ids_to_tokens(i): p
156
+ for i, p in id_logprob.items()
157
+ })
158
return logprobs
159
160
@@ -348,7 +353,7 @@ async def create_completion(raw_request: Request):
353
if request.suffix is not None:
354
# The language models we currently support do not support suffix.
355
return create_error_response(HTTPStatus.BAD_REQUEST,
351
- "suffix is not currently supported")
356
+ "suffix is not currently supported")
357
358
if request.logit_bias is not None:
359
# TODO: support logit_bias in vLLM engine.
@@ -387,22 +392,23 @@ async def create_completion(raw_request: Request):
392
except ValueError as e:
393
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
394
390
- result_generator = engine.generate(prompt, sampling_params,
391
- request_id)
395
+ result_generator = engine.generate(prompt, sampling_params, request_id)
396
397
# Similar to the OpenAI API, when n != best_of, we do not stream the
398
# results. In addition, we do not stream the results when use beam search.
395
- stream = (request.stream and
396
- (request.best_of is None or request.n == request.best_of) and
397
- not request.use_beam_search)
399
+ stream = (request.stream
400
+ and (request.best_of is None or request.n == request.best_of)
401
+ and not request.use_beam_search)
402
403
async def abort_request() -> None:
404
await engine.abort(request_id)
405
402
- def create_stream_response_json(index: int,
403
- text: str,
404
- logprobs: Optional[LogProbs] = None,
405
- finish_reason: Optional[str] = None) -> str:
406
+ def create_stream_response_json(
407
+ index: int,
408
+ text: str,
409
+ logprobs: Optional[LogProbs] = None,
410
+ finish_reason: Optional[str] = None,
411
+ ) -> str:
412
choice_data = CompletionResponseStreamChoice(
413
index=index,
414
text=text,
@@ -443,7 +449,8 @@ async def create_completion(raw_request: Request):
449
)
450
yield f"data: {response_json}\n\n"
451
if output.finish_reason is not None:
446
- logprobs = LogProbs() if request.logprobs is not None else None
452
+ logprobs = (LogProbs()
453
+ if request.logprobs is not None else None)
454
response_json = create_stream_response_json(
455
index=i,
456
text="",
@@ -487,8 +494,8 @@ async def create_completion(raw_request: Request):
494
choices.append(choice_data)
495
496
num_prompt_tokens = len(final_res.prompt_token_ids)
490
- num_generated_tokens = sum(len(output.token_ids)
491
- for output in final_res.outputs)
497
+ num_generated_tokens = sum(
498
+ len(output.token_ids) for output in final_res.outputs)
499
usage = UsageInfo(
500
prompt_tokens=num_prompt_tokens,
501
completion_tokens=num_generated_tokens,
@@ -506,9 +513,11 @@ async def create_completion(raw_request: Request):
513
# When user requests streaming but we don't stream, we still need to
514
# return a streaming response with a single event.
515
response_json = response.json(ensure_ascii=False)
516
+
517
async def fake_stream_generator() -> AsyncGenerator[str, None]:
518
yield f"data: {response_json}\n\n"
519
yield "data: [DONE]\n\n"
520
+
521
return StreamingResponse(fake_stream_generator(),
522
media_type="text/event-stream")
523
@@ -517,26 +526,34 @@ async def create_completion(raw_request: Request):
526
527
if __name__ == "__main__":
528
parser = argparse.ArgumentParser(
520
- description="vLLM OpenAI-Compatible RESTful API server."
521
- )
522
- parser.add_argument("--host", type=str, default="localhost", help="host name")
529
+ description="vLLM OpenAI-Compatible RESTful API server.")
530
+ parser.add_argument("--host",
531
+ type=str,
532
+ default="localhost",
533
+ help="host name")
534
parser.add_argument("--port", type=int, default=8000, help="port number")
535
+ parser.add_argument("--allow-credentials",
536
+ action="store_true",
537
+ help="allow credentials")
538
+ parser.add_argument("--allowed-origins",
539
+ type=json.loads,
540
+ default=["*"],
541
+ help="allowed origins")
542
+ parser.add_argument("--allowed-methods",
543
+ type=json.loads,
544
+ default=["*"],
545
+ help="allowed methods")
546
+ parser.add_argument("--allowed-headers",
547
+ type=json.loads,
548
+ default=["*"],
549
+ help="allowed headers")
550
parser.add_argument(
525
- "--allow-credentials", action="store_true", help="allow credentials"
526
- )
527
- parser.add_argument(
528
- "--allowed-origins", type=json.loads, default=["*"], help="allowed origins"
529
- )
530
- parser.add_argument(
531
- "--allowed-methods", type=json.loads, default=["*"], help="allowed methods"
532
- )
533
- parser.add_argument(
534
- "--allowed-headers", type=json.loads, default=["*"], help="allowed headers"
535
- )
536
- parser.add_argument("--served-model-name", type=str, default=None,
537
- help="The model name used in the API. If not specified, "
538
- "the model name will be the same as the "
539
- "huggingface name.")
551
+ "--served-model-name",
552
+ type=str,
553
+ default=None,
554
+ help="The model name used in the API. If not specified, "
555
+ "the model name will be the same as the "
556
+ "huggingface name.")
557
parser = AsyncEngineArgs.add_cli_args(parser)
558
args = parser.parse_args()
559
@@ -556,7 +573,11 @@ if __name__ == "__main__":
573
engine = AsyncLLMEngine.from_engine_args(engine_args)
574
575
# A separate tokenizer to map token IDs to strings.
559
- tokenizer = get_tokenizer(engine_args.tokenizer, engine_args.tokenizer_mode)
576
+ tokenizer = get_tokenizer(engine_args.tokenizer,
577
+ tokenizer_mode=engine_args.tokenizer_mode)
578
561
- uvicorn.run(app, host=args.host, port=args.port, log_level="info",
579
+ uvicorn.run(app,
580
+ host=args.host,
581
+ port=args.port,
582
+ log_level="info",
583
timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
vllm/entrypoints/openai/protocol.py
+4
-2
@@ -1,4 +1,5 @@
1
-# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
1
+# Adapted from
2
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3
import time
4
from typing import Dict, List, Literal, Optional, Union
5
@@ -98,7 +99,8 @@ class LogProbs(BaseModel):
99
text_offset: List[int] = Field(default_factory=list)
100
token_logprobs: List[Optional[float]] = Field(default_factory=list)
101
tokens: List[str] = Field(default_factory=list)
101
- top_logprobs: List[Optional[Dict[str, float]]] = Field(default_factory=list)
102
+ top_logprobs: List[Optional[Dict[str,
103
+ float]]] = Field(default_factory=list)
104
105
106
class CompletionResponseChoice(BaseModel):
vllm/logger.py
+3
-3
@@ -1,9 +1,9 @@
1
-# Adapted from https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
2
-
1
+# Adapted from
2
+# https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
3
+"""Logging configuration for vLLM."""
4
import logging
5
import sys
6
6
-
7
_FORMAT = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s"
8
_DATE_FORMAT = "%m-%d %H:%M:%S"
9
vllm/model_executor/__init__.py
-1
@@ -2,7 +2,6 @@ from vllm.model_executor.input_metadata import InputMetadata
2
from vllm.model_executor.model_loader import get_model
3
from vllm.model_executor.utils import set_random_seed
4
5
-
5
__all__ = [
6
"InputMetadata",
7
"get_model",
vllm/model_executor/input_metadata.py
+13
-2
@@ -8,11 +8,22 @@ from vllm.sequence import SequenceData
8
9
10
class InputMetadata:
11
+ """Metadata for input sequences. Used for PagedAttention.
12
+
13
+ Args:
14
+ seq_groups: List of (seq_ids, sampling_params).
15
+ seq_data: Seq_id -> SequenceData.
16
+ prompt_lens: Lengths of prompts.
17
+ slot_mapping: The address to write the new KV to of each token.
18
+ context_lens: the length of attention context for each generation token.
19
+ max_context_len: The maximum context length.
20
+ block_tables: The block tables. (Seq id -> list of physical block)
21
+ """
22
23
def __init__(
24
self,
14
- seq_groups: List[Tuple[List[int], SamplingParams]], # List of (seq_ids, sampling_params).
15
- seq_data: Dict[int, SequenceData], # Seq_id -> SequenceData.
25
+ seq_groups: List[Tuple[List[int], SamplingParams]],
26
+ seq_data: Dict[int, SequenceData],
27
prompt_lens: List[int],
28
slot_mapping: torch.Tensor,
29
context_lens: torch.Tensor,
vllm/model_executor/layers/activation.py
+9
-10
@@ -6,9 +6,10 @@ from vllm import activation_ops
6
7
_ACTIVATION_REGISTRY = {
8
"gelu": nn.GELU(),
9
- "gelu_new": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
10
- "gelu_fast": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
11
- "gelu_pytorch_tanh": nn.GELU(approximate="tanh"), # NOTE: This may introduce small rounding errors.
9
+ # NOTE: The following GELU functions may introduce small rounding errors.
10
+ "gelu_new": nn.GELU(approximate="tanh"),
11
+ "gelu_fast": nn.GELU(approximate="tanh"),
12
+ "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
13
"relu": nn.ReLU(),
14
}
15
@@ -25,15 +26,13 @@ class SiluAndMul(nn.Module):
26
"""An activation function for SwiGLU.
27
28
The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[1] // 2.
28
- """
29
30
- def __init__(self):
31
- super().__init__()
30
+ Shapes:
31
+ x: (num_tokens, 2 * d)
32
+ return: (num_tokens, d)
33
+ """
34
33
- def forward(
34
- self,
35
- x: torch.Tensor, # (num_tokens, 2 * d)
36
- ) -> torch.Tensor: # (num_tokens, d)
35
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
36
num_tokens = x.shape[0]
37
d = x.shape[1] // 2
38
out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
vllm/model_executor/layers/attention.py
+81
-32
@@ -14,6 +14,7 @@ _SUPPORTED_HEAD_SIZES = [64, 80, 96, 128]
14
15
16
class PagedAttention(nn.Module):
17
+ # pylint: disable=line-too-long
18
"""GPT-style multi-head PagedAttention.
19
20
This class takes flattened 1D query, key, and value tensors as input. The
@@ -54,12 +55,20 @@ class PagedAttention(nn.Module):
55
56
def multi_query_kv_attention(
57
self,
57
- output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
58
- query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
59
- key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
60
- value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
58
+ output: torch.Tensor,
59
+ query: torch.Tensor,
60
+ key: torch.Tensor,
61
+ value: torch.Tensor,
62
attn_bias: xops.AttentionBias,
63
) -> torch.Tensor:
64
+ """Normal attention for the prompt tokens.
65
+
66
+ Args:
67
+ output: shape = [num_prompt_tokens, num_heads, head_size]
68
+ query: shape = [num_prompt_tokens, num_heads, head_size]
69
+ key: shape = [num_prompt_tokens, num_heads, head_size]
70
+ value: shape = [num_prompt_tokens, num_heads, head_size]
71
+ """
72
# TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
73
out = xops.memory_efficient_attention_forward(
74
query.unsqueeze(0),
@@ -76,12 +85,22 @@ class PagedAttention(nn.Module):
85
86
def single_query_cached_kv_attention(
87
self,
79
- output: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
80
- query: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
81
- key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
82
- value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
88
+ output: torch.Tensor,
89
+ query: torch.Tensor,
90
+ key_cache: torch.Tensor,
91
+ value_cache: torch.Tensor,
92
input_metadata: InputMetadata,
93
) -> None:
94
+ """PagedAttention for the generation tokens.
95
+
96
+ Args:
97
+ output: shape = [num_generation_tokens, num_heads, head_size]
98
+ query: shape = [num_generation_tokens, num_heads, head_size]
99
+ key_cache: shape = [num_blocks, num_heads, head_size/x,
100
+ block_size, x]
101
+ value_cache: shape = [num_blocks, num_heads, head_size, block_size]
102
+ input_metadata: metadata for paged attention.
103
+ """
104
block_size = value_cache.shape[3]
105
attention_ops.single_query_cached_kv_attention(
106
output,
@@ -97,16 +116,32 @@ class PagedAttention(nn.Module):
116
117
def forward(
118
self,
100
- query: torch.Tensor, # [num_tokens, num_heads * head_size]
101
- key: torch.Tensor, # [num_tokens, num_heads * head_size]
102
- value: torch.Tensor, # [num_tokens, num_heads * head_size]
103
- key_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size/x, block_size, x]
104
- value_cache: Optional[torch.Tensor], # [num_blocks, num_heads, head_size, block_size]
119
+ query: torch.Tensor,
120
+ key: torch.Tensor,
121
+ value: torch.Tensor,
122
+ key_cache: Optional[torch.Tensor],
123
+ value_cache: Optional[torch.Tensor],
124
input_metadata: InputMetadata,
125
cache_event: Optional[torch.cuda.Event],
107
- ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
108
- # NOTE: The query, key, and value tensors must be sliced from a qkv
109
- # tensor of shape [num_tokens, 3 * num_heads * head_size].
126
+ ) -> torch.Tensor:
127
+ """PagedAttention forward pass.
128
+
129
+ NOTE: The query, key, and value tensors must be sliced from a qkv
130
+ tensor of shape [num_tokens, 3 * num_heads * head_size].
131
+
132
+ Args:
133
+ query: shape = [num_tokens, num_heads * head_size]
134
+ key: shape = [num_tokens, num_heads * head_size]
135
+ value: shape = [num_tokens, num_heads * head_size]
136
+ key_cache: shape = [num_blocks, num_heads, head_size/x,
137
+ block_size, x]
138
+ value_cache: shape = [num_blocks, num_heads, head_size, block_size]
139
+ input_metadata: metadata for paged attention.
140
+ cache_event: event to wait for the cache operations to finish.
141
+
142
+ Returns:
143
+ shape = [num_tokens, num_heads * head_size]
144
+ """
145
146
# Reshape the query, key, and value tensors.
147
query = query.view(-1, self.num_heads, self.head_size)
@@ -136,7 +171,7 @@ class PagedAttention(nn.Module):
171
# and value vectors will not be cached.
172
num_valid_tokens = input_metadata.num_valid_tokens
173
if (num_valid_tokens > 0 and key_cache is not None
139
- and value_cache is not None):
174
+ and value_cache is not None):
175
# The stride is 3 because the key and value are sliced from qkv.
176
cache_ops.reshape_and_cache(
177
key[:num_valid_tokens],
@@ -149,15 +184,12 @@ class PagedAttention(nn.Module):
184
if input_metadata.num_generation_tokens > 0:
185
assert key_cache is not None and value_cache is not None, (
186
"key_cache and value_cache must be provided when "
152
- "generating tokens."
153
- )
187
+ "generating tokens.")
188
# Compute the attention op for generation tokens.
189
self.single_query_cached_kv_attention(
190
output[num_prompt_tokens:num_valid_tokens],
157
- query[num_prompt_tokens:num_valid_tokens],
158
- key_cache,
159
- value_cache,
160
- input_metadata)
191
+ query[num_prompt_tokens:num_valid_tokens], key_cache,
192
+ value_cache, input_metadata)
193
194
# Reshape the output tensor.
195
# NOTE(woosuk): The output tensor may include paddings.
@@ -179,9 +211,9 @@ class PagedAttentionWithRoPE(PagedAttention):
211
super().__init__(num_heads, head_size, scale)
212
213
# Create the cos and sin cache.
182
- inv_freq = 1.0 / (base ** (torch.arange(0, rotary_dim, 2) / rotary_dim))
214
+ inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
215
t = torch.arange(max_position).float()
184
- freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
216
+ freqs = torch.einsum("i,j -> ij", t, inv_freq.float())
217
cos = freqs.cos()
218
sin = freqs.sin()
219
cache = torch.cat((cos, sin), dim=-1)
@@ -195,15 +227,32 @@ class PagedAttentionWithRoPE(PagedAttention):
227
228
def forward(
229
self,
198
- positions: torch.Tensor, # [num_tokens]
199
- query: torch.Tensor, # [num_tokens, num_heads * head_size]
200
- key: torch.Tensor, # [num_tokens, num_heads * head_size]
201
- value: torch.Tensor, # [num_tokens, num_heads * head_size]
202
- key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
203
- value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
230
+ positions: torch.Tensor,
231
+ query: torch.Tensor,
232
+ key: torch.Tensor,
233
+ value: torch.Tensor,
234
+ key_cache: torch.Tensor,
235
+ value_cache: torch.Tensor,
236
input_metadata: InputMetadata,
237
cache_event: Optional[torch.cuda.Event],
206
- ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
238
+ ) -> torch.Tensor:
239
+ """ PagedAttention forward pass with rotary embedding.
240
+
241
+ Args:
242
+ positions: shape = [num_tokens]
243
+ query: shape = [num_tokens, num_heads * head_size]
244
+ key: shape = [num_tokens, num_heads * head_size]
245
+ value: shape = [num_tokens, num_heads * head_size]
246
+ key_cache: shape = [num_blocks, num_heads, head_size/x,
247
+ block_size, x]
248
+ value_cache: shape = [num_blocks, num_heads, head_size, block_size]
249
+ input_metadata: metadata for paged attention.
250
+ cache_event: event to wait for the cache operations to finish.
251
+
252
+ Returns:
253
+ shape = [num_tokens, num_heads * head_size]
254
+ """
255
+
256
# Apply rotary embedding to the query and key before passing them
257
# to the attention op.
258
pos_encoding_ops.rotary_embedding_neox(
vllm/model_executor/layers/sampler.py
+45
-37
@@ -13,6 +13,7 @@ from vllm.sequence import SequenceOutputs
13
14
_SAMPLING_EPS = 1e-5
15
16
+
17
class Sampler(nn.Module):
18
"""Samples the next tokens from the model's outputs.
19
@@ -50,19 +51,20 @@ class Sampler(nn.Module):
51
# Apply presence and frequency penalties.
52
output_tokens = _get_output_tokens(input_metadata)
53
assert len(output_tokens) == logits.shape[0]
53
- presence_penalties, frequency_penalties = _get_penalties(input_metadata)
54
+ presence_penalties, frequency_penalties = _get_penalties(
55
+ input_metadata)
56
assert len(presence_penalties) == logits.shape[0]
57
assert len(frequency_penalties) == logits.shape[0]
56
- logits = _apply_penalties(
57
- logits, output_tokens, presence_penalties, frequency_penalties,
58
- self.vocab_size)
58
+ logits = _apply_penalties(logits, output_tokens, presence_penalties,
59
+ frequency_penalties, self.vocab_size)
60
61
# Apply temperature scaling.
62
temperatures = _get_temperatures(input_metadata)
63
assert len(temperatures) == logits.shape[0]
64
if any(t != 1.0 for t in temperatures):
64
- t = torch.tensor(
65
- temperatures, dtype=logits.dtype, device=logits.device)
65
+ t = torch.tensor(temperatures,
66
+ dtype=logits.dtype,
67
+ device=logits.device)
68
# Use in-place division to avoid creating a new tensor.
69
logits.div_(t.unsqueeze(dim=1))
70
@@ -75,7 +77,9 @@ class Sampler(nn.Module):
77
# Apply top-p and top-k truncation.
78
top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
79
assert len(top_ps) == len(top_ks) == probs.shape[0]
78
- if any(p < 1.0 - _SAMPLING_EPS for p in top_ps) or any(k != self.vocab_size for k in top_ks):
80
+ do_top_p = any(p < 1.0 - _SAMPLING_EPS for p in top_ps)
81
+ do_top_k = any(k != self.vocab_size for k in top_ks)
82
+ if do_top_p or do_top_k:
83
probs = _apply_top_p_top_k(probs, top_ps, top_ks)
84
85
# Sample the next tokens.
@@ -97,8 +101,7 @@ def _prune_hidden_states(
101
102
103
def _get_penalties(
100
- input_metadata: InputMetadata,
101
-) -> Tuple[List[float], List[float]]:
104
+ input_metadata: InputMetadata) -> Tuple[List[float], List[float]]:
105
# Collect the presence and frequency penalties.
106
presence_penalties: List[float] = []
107
frequency_penalties: List[float] = []
@@ -117,9 +120,7 @@ def _get_penalties(
120
return presence_penalties, frequency_penalties
121
122
120
-def _get_output_tokens(
121
- input_metadata: InputMetadata,
122
-) -> List[List[int]]:
123
+def _get_output_tokens(input_metadata: InputMetadata) -> List[List[int]]:
124
output_tokens: List[List[int]] = []
125
for i, seq_group in enumerate(input_metadata.seq_groups):
126
seq_ids, _ = seq_group
@@ -169,11 +170,13 @@ def _apply_penalties(
170
device=logits.device)
171
172
frequency_penalties = [frequency_penalties[i] for i in indices]
172
- frequency_penalties = torch.tensor(
173
- frequency_penalties, dtype=logits.dtype, device=logits.device)
173
+ frequency_penalties = torch.tensor(frequency_penalties,
174
+ dtype=logits.dtype,
175
+ device=logits.device)
176
presence_penalties = [presence_penalties[i] for i in indices]
175
- presence_penalties = torch.tensor(
176
- presence_penalties, dtype=logits.dtype, device=logits.device)
177
+ presence_penalties = torch.tensor(presence_penalties,
178
+ dtype=logits.dtype,
179
+ device=logits.device)
180
181
# We follow the definition in OpenAI API.
182
# Refer to https://platform.openai.com/docs/api-reference/parameter-details
@@ -183,9 +186,7 @@ def _apply_penalties(
186
return logits
187
188
186
-def _get_temperatures(
187
- input_metadata: InputMetadata,
188
-) -> List[float]:
189
+def _get_temperatures(input_metadata: InputMetadata) -> List[float]:
190
# Collect the temperatures for the logits.
191
temperatures: List[float] = []
192
for i, seq_group in enumerate(input_metadata.seq_groups):
@@ -252,8 +253,9 @@ def _apply_top_p_top_k(
253
probs_sort[top_k_mask] = 0.0
254
255
# Re-sort the probabilities.
255
- probs = torch.gather(
256
- probs_sort, dim=-1, index=torch.argsort(probs_idx, dim=-1))
256
+ probs = torch.gather(probs_sort,
257
+ dim=-1,
258
+ index=torch.argsort(probs_idx, dim=-1))
259
return probs
260
261
@@ -296,8 +298,9 @@ def _sample_from_prompt(
298
# Random sampling.
299
# Sample `best_of` tokens for the prompt.
300
num_seqs = sampling_params.best_of
299
- next_token_ids = torch.multinomial(
300
- prob, num_samples=num_seqs, replacement=True)
301
+ next_token_ids = torch.multinomial(prob,
302
+ num_samples=num_seqs,
303
+ replacement=True)
304
next_token_ids = next_token_ids.tolist()
305
return next_token_ids
306
@@ -315,8 +318,9 @@ def _sample_from_generation_tokens(
318
if sampling_params.use_beam_search:
319
# Beam search.
320
# Add cumulative logprobs for the sequences in the group.
318
- seq_logprobs = torch.tensor(
319
- seq_logprobs, dtype=torch.float, device=logprobs.device)
321
+ seq_logprobs = torch.tensor(seq_logprobs,
322
+ dtype=torch.float,
323
+ device=logprobs.device)
324
logprobs = logprobs + seq_logprobs.unsqueeze(dim=1)
325
326
vocab_size = logprobs.size(-1)
@@ -353,8 +357,9 @@ def _sample_from_generation_tokens(
357
else:
358
# Random sampling.
359
# Sample 1 token for each sequence in the group.
356
- next_token_ids = torch.multinomial(
357
- probs, num_samples=1, replacement=True)
360
+ next_token_ids = torch.multinomial(probs,
361
+ num_samples=1,
362
+ replacement=True)
363
next_token_ids = next_token_ids.squeeze(dim=-1).tolist()
364
parent_seq_ids = seq_ids
365
return parent_seq_ids, next_token_ids
@@ -381,15 +386,16 @@ def _sample(
386
# Sample the next tokens.
387
next_token_ids = _sample_from_prompt(prob, sampling_params)
388
# Get top-k log probabilities for the next tokens.
384
- next_logprobs = _get_topk_logprobs(
385
- logprob, sampling_params.logprobs)
389
+ next_logprobs = _get_topk_logprobs(logprob,
390
+ sampling_params.logprobs)
391
392
# Build the output.
393
for seq_id, next_token_id in zip(seq_ids, next_token_ids):
394
output_logprobs = next_logprobs.copy()
395
output_logprobs[next_token_id] = logprob[next_token_id].item()
391
- seq_outputs[seq_id] = SequenceOutputs(
392
- seq_id, seq_id, next_token_id, output_logprobs)
396
+ seq_outputs[seq_id] = SequenceOutputs(seq_id, seq_id,
397
+ next_token_id,
398
+ output_logprobs)
399
else:
400
# Generate the next tokens for generation tokens.
401
prob = probs[idx:idx + len(seq_ids)]
@@ -399,22 +405,24 @@ def _sample(
405
# Sample the next tokens.
406
seq_logprobs = [
407
input_metadata.seq_data[seq_id].cumulative_logprob
402
- for seq_id in seq_ids]
408
+ for seq_id in seq_ids
409
+ ]
410
parent_seq_ids, next_token_ids = _sample_from_generation_tokens(
411
seq_ids, prob, logprob, seq_logprobs, sampling_params)
412
413
# Get top-k log probabilities for the next tokens.
414
next_logprobs: Dict[int, Dict[int, float]] = {}
408
- for i, seq_id in enumerate(seq_ids):
415
+ for j, seq_id in enumerate(seq_ids):
416
next_logprobs[seq_id] = _get_topk_logprobs(
410
- logprob[i], sampling_params.logprobs)
417
+ logprob[j], sampling_params.logprobs)
418
419
# Build the output.
420
for seq_id, parent_seq_id, next_token_id in zip(
414
- seq_ids, parent_seq_ids, next_token_ids):
415
- i = seq_ids.index(parent_seq_id)
421
+ seq_ids, parent_seq_ids, next_token_ids):
422
+ j = seq_ids.index(parent_seq_id)
423
output_logprobs = next_logprobs[parent_seq_id].copy()
417
- output_logprobs[next_token_id] = logprob[i, next_token_id].item()
424
+ output_logprobs[next_token_id] = logprob[j,
425
+ next_token_id].item()
426
seq_outputs[seq_id] = SequenceOutputs(
427
seq_id,
428
parent_seq_id,
vllm/model_executor/model_loader.py
+6
-7
@@ -6,8 +6,9 @@ import torch.nn as nn
6
from transformers import PretrainedConfig
7
8
from vllm.config import ModelConfig
9
-from vllm.model_executor.models import (GPT2LMHeadModel, GPTBigCodeForCausalLM, GPTNeoXForCausalLM,
10
- LlamaForCausalLM, OPTForCausalLM)
9
+from vllm.model_executor.models import (GPT2LMHeadModel, GPTBigCodeForCausalLM,
10
+ GPTNeoXForCausalLM, LlamaForCausalLM,
11
+ OPTForCausalLM)
12
from vllm.model_executor.weight_utils import initialize_dummy_weights
13
14
# TODO(woosuk): Lazy-load the model classes.
@@ -28,8 +29,7 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
29
return _MODEL_REGISTRY[arch]
30
raise ValueError(
31
f"Model architectures {architectures} are not supported for now. "
31
- f"Supported architectures: {list(_MODEL_REGISTRY.keys())}"
32
- )
32
+ f"Supported architectures: {list(_MODEL_REGISTRY.keys())}")
33
34
35
def get_model(model_config: ModelConfig) -> nn.Module:
@@ -46,8 +46,7 @@ def get_model(model_config: ModelConfig) -> nn.Module:
46
initialize_dummy_weights(model)
47
else:
48
# Load the weights from the cached or downloaded files.
49
- model.load_weights(
50
- model_config.model, model_config.download_dir,
51
- model_config.use_np_weights)
49
+ model.load_weights(model_config.model, model_config.download_dir,
50
+ model_config.use_np_weights)
51
model = model.cuda()
52
return model.eval()
vllm/model_executor/models/__init__.py
-2
@@ -4,8 +4,6 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
4
from vllm.model_executor.models.llama import LlamaForCausalLM
5
from vllm.model_executor.models.opt import OPTForCausalLM
6
7
-
8
-
7
__all__ = [
8
"GPT2LMHeadModel",
9
"GPTBigCodeForCausalLM",
vllm/model_executor/models/gpt2.py
+53
-33
@@ -1,5 +1,6 @@
1
# coding=utf-8
2
-# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
4
# Copyright 2023 The vLLM team.
5
# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
6
# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
@@ -47,19 +48,25 @@ class GPT2Attention(nn.Module):
48
super().__init__()
49
self.hidden_size = config.hidden_size
50
total_num_heads = config.num_attention_heads
50
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
51
+ tensor_model_parallel_world_size = (
52
+ get_tensor_model_parallel_world_size())
53
assert total_num_heads % tensor_model_parallel_world_size == 0
54
self.num_heads = total_num_heads // tensor_model_parallel_world_size
55
self.head_dim = self.hidden_size // total_num_heads
54
- self.scale = self.head_dim ** -0.5
56
+ self.scale = self.head_dim**-0.5
57
56
- self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size,
57
- bias=True, gather_output=False,
58
+ self.c_attn = ColumnParallelLinear(self.hidden_size,
59
+ 3 * self.hidden_size,
60
+ bias=True,
61
+ gather_output=False,
62
perform_initialization=False)
59
- self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size,
60
- bias=True, input_is_parallel=True,
63
+ self.c_proj = RowParallelLinear(self.hidden_size,
64
+ self.hidden_size,
65
+ bias=True,
66
+ input_is_parallel=True,
67
perform_initialization=False)
62
- self.attn = PagedAttention(self.num_heads, self.head_dim,
68
+ self.attn = PagedAttention(self.num_heads,
69
+ self.head_dim,
70
scale=self.scale)
71
72
def forward(
@@ -72,8 +79,8 @@ class GPT2Attention(nn.Module):
79
qkv, _ = self.c_attn(hidden_states)
80
q, k, v = qkv.chunk(chunks=3, dim=-1)
81
key_cache, value_cache = kv_cache
75
- attn_output = self.attn(
76
- q, k, v, key_cache, value_cache, input_metadata, cache_event)
82
+ attn_output = self.attn(q, k, v, key_cache, value_cache,
83
+ input_metadata, cache_event)
84
attn_output, _ = self.c_proj(attn_output)
85
return attn_output
86
@@ -87,11 +94,15 @@ class GPT2MLP(nn.Module):
94
):
95
super().__init__()
96
hidden_size = config.hidden_size
90
- self.c_fc = ColumnParallelLinear(hidden_size, intermediate_size,
91
- bias=True, gather_output=False,
97
+ self.c_fc = ColumnParallelLinear(hidden_size,
98
+ intermediate_size,
99
+ bias=True,
100
+ gather_output=False,
101
perform_initialization=False)
93
- self.c_proj = RowParallelLinear(intermediate_size, hidden_size,
94
- bias=True, input_is_parallel=True,
102
+ self.c_proj = RowParallelLinear(intermediate_size,
103
+ hidden_size,
104
+ bias=True,
105
+ input_is_parallel=True,
106
perform_initialization=False)
107
self.act = get_act_fn(config.activation_function)
108
@@ -107,7 +118,8 @@ class GPT2Block(nn.Module):
118
def __init__(self, config: GPT2Config):
119
super().__init__()
120
hidden_size = config.hidden_size
110
- inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
121
+ inner_dim = (config.n_inner if config.n_inner is not None else 4 *
122
+ hidden_size)
123
124
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
125
self.attn = GPT2Attention(config)
@@ -145,9 +157,9 @@ class GPT2Model(nn.Module):
157
def __init__(self, config: GPT2Config):
158
super().__init__()
159
self.config = config
148
- assert config.add_cross_attention == False
149
- assert config.scale_attn_by_inverse_layer_idx == False
150
- assert config.reorder_and_upcast_attn == False
160
+ assert not config.add_cross_attention
161
+ assert not config.scale_attn_by_inverse_layer_idx
162
+ assert not config.reorder_and_upcast_attn
163
self.embed_dim = config.hidden_size
164
165
# Optimization: While the vocab size of GPT-2 is 50257, we extend it
@@ -180,8 +192,8 @@ class GPT2Model(nn.Module):
192
else:
193
cache_event = cache_events[i]
194
layer = self.h[i]
183
- hidden_states = layer(
184
- hidden_states, kv_caches[i], input_metadata, cache_event)
195
+ hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
196
+ cache_event)
197
198
hidden_states = self.ln_f(hidden_states)
199
return hidden_states
@@ -206,24 +218,26 @@ class GPT2LMHeadModel(nn.Module):
218
input_metadata: InputMetadata,
219
cache_events: Optional[List[torch.cuda.Event]],
220
) -> Dict[int, SequenceOutputs]:
209
- hidden_states = self.transformer(
210
- input_ids, positions, kv_caches, input_metadata, cache_events)
211
- next_tokens = self.sampler(
212
- self.lm_head_weight, hidden_states, input_metadata)
221
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
222
+ input_metadata, cache_events)
223
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
224
+ input_metadata)
225
return next_tokens
226
227
_column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
228
_row_parallel_weights = ["c_proj.weight"]
229
218
- def load_weights(self, model_name_or_path: str,
230
+ def load_weights(self,
231
+ model_name_or_path: str,
232
cache_dir: Optional[str] = None,
233
use_np_cache: bool = False):
221
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
234
+ tensor_model_parallel_world_size = (
235
+ get_tensor_model_parallel_world_size())
236
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
237
state_dict = self.state_dict()
238
239
for name, loaded_weight in hf_model_weights_iterator(
226
- model_name_or_path, cache_dir, use_np_cache):
240
+ model_name_or_path, cache_dir, use_np_cache):
241
if "lm_head.weight" in name:
242
# GPT-2 ties the weights of the embedding layer and the final
243
# linear layer.
@@ -248,16 +262,20 @@ class GPT2LMHeadModel(nn.Module):
262
263
if name == "transformer.wte.weight":
264
# Consider padding in the vocab size.
251
- padded_vocab_size = param.shape[0] * tensor_model_parallel_world_size
265
+ padded_vocab_size = (param.shape[0] *
266
+ tensor_model_parallel_world_size)
267
num_extra_rows = padded_vocab_size - self.config.vocab_size
253
- extra_rows = torch.empty(num_extra_rows, loaded_weight.shape[1])
268
+ extra_rows = torch.empty(num_extra_rows,
269
+ loaded_weight.shape[1])
270
extra_rows = extra_rows.to(loaded_weight)
271
loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
272
273
# For the fused QKV linear layer, manually shard the weights.
274
if "c_attn" in name:
259
- # GPT-2's fused QKV has the shape of [3 * num_heads * head_size, hidden_size].
260
- # When tensor parallelism is used, we shard the weights along the head dimension.
275
+ # GPT-2's fused QKV has the shape of
276
+ # [3 * num_heads * head_size, hidden_size].
277
+ # When tensor parallelism is used, we shard the weights along
278
+ # the head dimension.
279
total_num_heads = self.config.num_attention_heads
280
hidden_size = self.config.hidden_size
281
head_size = hidden_size // total_num_heads
@@ -266,11 +284,13 @@ class GPT2LMHeadModel(nn.Module):
284
head_end = (tensor_model_parallel_rank + 1) * num_heads
285
286
if name.endswith(".weight"):
269
- loaded_weight = loaded_weight.view(3, total_num_heads, head_size, hidden_size)
287
+ loaded_weight = loaded_weight.view(3, total_num_heads,
288
+ head_size, hidden_size)
289
loaded_weight = loaded_weight[:, head_start:head_end, :, :]
290
loaded_weight = loaded_weight.reshape(-1, hidden_size)
291
elif name.endswith(".bias"):
273
- loaded_weight = loaded_weight.view(3, total_num_heads, head_size)
292
+ loaded_weight = loaded_weight.view(3, total_num_heads,
293
+ head_size)
294
loaded_weight = loaded_weight[:, head_start:head_end, :]
295
loaded_weight = loaded_weight.reshape(-1)
296
else:
vllm/model_executor/models/gpt_bigcode.py
+66
-38
@@ -1,5 +1,6 @@
1
# coding=utf-8
2
-# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt2/modeling_gpt2.py
4
# Copyright 2023 The vLLM team.
5
# Copyright 2023 CTranslate2, and Michael Feil
6
# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team.
@@ -49,19 +50,25 @@ class GPTBigCodeAttention(nn.Module):
50
super().__init__()
51
self.hidden_size = config.hidden_size
52
total_num_heads = config.num_attention_heads
52
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
53
+ tensor_model_parallel_world_size = (
54
+ get_tensor_model_parallel_world_size())
55
assert total_num_heads % tensor_model_parallel_world_size == 0
56
self.num_heads = total_num_heads // tensor_model_parallel_world_size
57
self.head_dim = self.hidden_size // total_num_heads
56
- self.scale = self.head_dim ** -0.5
58
+ self.scale = self.head_dim**-0.5
59
58
- self.c_attn = ColumnParallelLinear(self.hidden_size, 3 * self.hidden_size,
59
- bias=True, gather_output=False,
60
+ self.c_attn = ColumnParallelLinear(self.hidden_size,
61
+ 3 * self.hidden_size,
62
+ bias=True,
63
+ gather_output=False,
64
perform_initialization=False)
61
- self.c_proj = RowParallelLinear(self.hidden_size, self.hidden_size,
62
- bias=True, input_is_parallel=True,
65
+ self.c_proj = RowParallelLinear(self.hidden_size,
66
+ self.hidden_size,
67
+ bias=True,
68
+ input_is_parallel=True,
69
perform_initialization=False)
64
- self.attn = PagedAttention(self.num_heads, self.head_dim,
70
+ self.attn = PagedAttention(self.num_heads,
71
+ self.head_dim,
72
scale=self.scale)
73
74
def forward(
@@ -74,8 +81,8 @@ class GPTBigCodeAttention(nn.Module):
81
qkv, _ = self.c_attn(hidden_states)
82
q, k, v = qkv.chunk(chunks=3, dim=-1)
83
key_cache, value_cache = kv_cache
77
- attn_output = self.attn(
78
- q, k, v, key_cache, value_cache, input_metadata, cache_event)
84
+ attn_output = self.attn(q, k, v, key_cache, value_cache,
85
+ input_metadata, cache_event)
86
attn_output, _ = self.c_proj(attn_output)
87
return attn_output
88
@@ -89,11 +96,15 @@ class GPTBigMLP(nn.Module):
96
):
97
super().__init__()
98
hidden_size = config.hidden_size
92
- self.c_fc = ColumnParallelLinear(hidden_size, intermediate_size,
93
- bias=True, gather_output=False,
99
+ self.c_fc = ColumnParallelLinear(hidden_size,
100
+ intermediate_size,
101
+ bias=True,
102
+ gather_output=False,
103
perform_initialization=False)
95
- self.c_proj = RowParallelLinear(intermediate_size, hidden_size,
96
- bias=True, input_is_parallel=True,
104
+ self.c_proj = RowParallelLinear(intermediate_size,
105
+ hidden_size,
106
+ bias=True,
107
+ input_is_parallel=True,
108
perform_initialization=False)
109
self.act = get_act_fn(config.activation_function)
110
@@ -109,7 +120,8 @@ class GPTBigCodeBlock(nn.Module):
120
def __init__(self, config: GPTBigCodeConfig):
121
super().__init__()
122
hidden_size = config.hidden_size
112
- inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
123
+ inner_dim = (config.n_inner if config.n_inner is not None else 4 *
124
+ hidden_size)
125
126
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
127
self.attn = GPTBigCodeAttention(config)
@@ -147,7 +159,7 @@ class GPTBigCodeModel(nn.Module):
159
def __init__(self, config: GPTBigCodeConfig):
160
super().__init__()
161
self.config = config
150
- assert config.add_cross_attention == False
162
+ assert not config.add_cross_attention
163
164
self.embed_dim = config.hidden_size
165
@@ -181,8 +193,8 @@ class GPTBigCodeModel(nn.Module):
193
else:
194
cache_event = cache_events[i]
195
layer = self.h[i]
184
- hidden_states = layer(
185
- hidden_states, kv_caches[i], input_metadata, cache_event)
196
+ hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
197
+ cache_event)
198
199
hidden_states = self.ln_f(hidden_states)
200
return hidden_states
@@ -207,24 +219,26 @@ class GPTBigCodeForCausalLM(nn.Module):
219
input_metadata: InputMetadata,
220
cache_events: Optional[List[torch.cuda.Event]],
221
) -> Dict[int, SequenceOutputs]:
210
- hidden_states = self.transformer(
211
- input_ids, positions, kv_caches, input_metadata, cache_events)
212
- next_tokens = self.sampler(
213
- self.lm_head_weight, hidden_states, input_metadata)
222
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
223
+ input_metadata, cache_events)
224
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
225
+ input_metadata)
226
return next_tokens
227
228
_column_parallel_weights = ["wte.weight", "c_fc.weight", "c_fc.bias"]
229
_row_parallel_weights = ["c_proj.weight"]
230
219
- def load_weights(self, model_name_or_path: str,
231
+ def load_weights(self,
232
+ model_name_or_path: str,
233
cache_dir: Optional[str] = None,
234
use_np_cache: bool = False):
222
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
235
+ tensor_model_parallel_world_size = (
236
+ get_tensor_model_parallel_world_size())
237
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
238
state_dict = self.state_dict()
239
240
for name, loaded_weight in hf_model_weights_iterator(
227
- model_name_or_path, cache_dir, use_np_cache):
241
+ model_name_or_path, cache_dir, use_np_cache):
242
if "lm_head.weight" in name:
243
# GPT-2 ties the weights of the embedding layer and the final
244
# linear layer.
@@ -241,9 +255,11 @@ class GPTBigCodeForCausalLM(nn.Module):
255
256
if name == "transformer.wte.weight":
257
# Consider padding in the vocab size.
244
- padded_vocab_size = param.shape[0] * tensor_model_parallel_world_size
258
+ padded_vocab_size = param.shape[
259
+ 0] * tensor_model_parallel_world_size
260
num_extra_rows = padded_vocab_size - self.config.vocab_size
246
- extra_rows = torch.empty(num_extra_rows, loaded_weight.shape[1])
261
+ extra_rows = torch.empty(num_extra_rows,
262
+ loaded_weight.shape[1])
263
extra_rows = extra_rows.to(loaded_weight)
264
loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
265
@@ -258,25 +274,31 @@ class GPTBigCodeForCausalLM(nn.Module):
274
qkv_array = qkv_array.numpy()
275
276
dims_q = n_head * head_dim
261
- q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim), axis=0)
262
- # q is fine, but k & v have not replicated shape along the first axis
263
- # as long as MQA is not nativly supported, increase memory and replicated
264
- # (head_dim, hidden_dim) to (n_heads * head_dim, hidden_dim)
277
+ # pylint: disable=unbalanced-tuple-unpacking
278
+ q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim),
279
+ axis=0)
280
+ # q is fine, but k & v have not replicated shape along the first
281
+ # axis as long as MQA is not nativly supported, increase memory
282
+ # and replicated (head_dim, hidden_dim) to
283
+ # (n_heads * head_dim, hidden_dim)
284
if k.ndim == 2 and v.ndim == 2:
285
replication = (n_head, 1) # weights
286
else:
287
replication = n_head # biases
288
# replicate n_head times for q, v
289
k, v = np.tile(k, replication), np.tile(v, replication)
271
- # concat q, k, v along the first axis (n_heads * head_dim, hidden_dim)
290
+ # concat q, k, v along the first axis
291
+ # (n_heads * head_dim, hidden_dim)
292
# to (3 * n_heads * head_dim, hidden_dim)
293
qkv_array = np.concatenate((q, k, v), axis=0)
294
return torch.from_numpy(qkv_array)
295
296
# For the fused QKV linear layer, manually shard the weights.
297
if "c_attn" in name:
278
- # GPT-2's fused QKV has the shape of [3 * num_heads * head_size, hidden_size].
279
- # When tensor parallelism is used, we shard the weights along the head dimension.
298
+ # GPT-2's fused QKV has the shape of
299
+ # [3 * num_heads * head_size, hidden_size].
300
+ # When tensor parallelism is used, we shard the weights along
301
+ # the head dimension.
302
total_num_heads = self.config.num_attention_heads
303
hidden_size = self.config.hidden_size
304
head_size = hidden_size // total_num_heads
@@ -285,13 +307,19 @@ class GPTBigCodeForCausalLM(nn.Module):
307
head_end = (tensor_model_parallel_rank + 1) * num_heads
308
309
if name.endswith(".weight"):
288
- loaded_weight = _expand_mqa_mha(loaded_weight, n_head=total_num_heads, head_dim=head_size)
289
- loaded_weight = loaded_weight.view(3, total_num_heads, head_size, hidden_size)
310
+ loaded_weight = _expand_mqa_mha(loaded_weight,
311
+ n_head=total_num_heads,
312
+ head_dim=head_size)
313
+ loaded_weight = loaded_weight.view(3, total_num_heads,
314
+ head_size, hidden_size)
315
loaded_weight = loaded_weight[:, head_start:head_end, :, :]
316
loaded_weight = loaded_weight.reshape(-1, hidden_size)
317
elif name.endswith(".bias"):
293
- loaded_weight = _expand_mqa_mha(loaded_weight, n_head=total_num_heads, head_dim=head_size)
294
- loaded_weight = loaded_weight.view(3, total_num_heads, head_size)
318
+ loaded_weight = _expand_mqa_mha(loaded_weight,
319
+ n_head=total_num_heads,
320
+ head_dim=head_size)
321
+ loaded_weight = loaded_weight.view(3, total_num_heads,
322
+ head_size)
323
loaded_weight = loaded_weight[:, head_start:head_end, :]
324
loaded_weight = loaded_weight.reshape(-1)
325
else:
vllm/model_executor/models/gpt_neox.py
+52
-33
@@ -1,5 +1,6 @@
1
# coding=utf-8
2
-# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
4
# Copyright 2023 The vLLM team.
5
# Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved.
6
#
@@ -48,19 +49,23 @@ class GPTNeoXAttention(nn.Module):
49
self.hidden_size = config.hidden_size
50
self.head_size = self.hidden_size // self.total_num_heads
51
51
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
52
+ tensor_model_parallel_world_size = (
53
+ get_tensor_model_parallel_world_size())
54
assert self.total_num_heads % tensor_model_parallel_world_size == 0
53
- self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
54
-
55
- self.query_key_value = ColumnParallelLinear(config.hidden_size,
56
- 3 * config.hidden_size,
57
- gather_output=False,
58
- perform_initialization=False)
59
- self.dense = RowParallelLinear(config.hidden_size, config.hidden_size,
55
+ self.num_heads = (self.total_num_heads //
56
+ tensor_model_parallel_world_size)
57
+
58
+ self.query_key_value = ColumnParallelLinear(
59
+ config.hidden_size,
60
+ 3 * config.hidden_size,
61
+ gather_output=False,
62
+ perform_initialization=False)
63
+ self.dense = RowParallelLinear(config.hidden_size,
64
+ config.hidden_size,
65
input_is_parallel=True,
66
perform_initialization=False)
67
63
- scaling = self.head_size ** -0.5
68
+ scaling = self.head_size**-0.5
69
rotary_dim = int(self.head_size * config.rotary_pct)
70
assert rotary_dim % 2 == 0
71
self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
@@ -78,8 +83,8 @@ class GPTNeoXAttention(nn.Module):
83
84
q, k, v = qkv.chunk(chunks=3, dim=-1)
85
k_cache, v_cache = kv_cache
81
- attn_output = self.attn(
82
- position_ids, q, k, v, k_cache, v_cache, input_metadata, cache_event)
86
+ attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
87
+ input_metadata, cache_event)
88
output, _ = self.dense(attn_output)
89
return output
90
@@ -92,7 +97,8 @@ class GPTNeoXMLP(nn.Module):
97
config.intermediate_size,
98
gather_output=False,
99
perform_initialization=False)
95
- self.dense_4h_to_h = RowParallelLinear(config.intermediate_size, config.hidden_size,
100
+ self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
101
+ config.hidden_size,
102
input_is_parallel=True,
103
perform_initialization=False)
104
self.act = get_act_fn(config.hidden_act)
@@ -109,8 +115,10 @@ class GPTNeoXLayer(nn.Module):
115
def __init__(self, config: GPTNeoXConfig):
116
super().__init__()
117
self.use_parallel_residual = config.use_parallel_residual
112
- self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
113
- self.post_attention_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
118
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
119
+ eps=config.layer_norm_eps)
120
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
121
+ eps=config.layer_norm_eps)
122
self.attention = GPTNeoXAttention(config)
123
self.mlp = GPTNeoXMLP(config)
124
@@ -154,10 +162,13 @@ class GPTNeoXModel(nn.Module):
162
super().__init__()
163
self.config = config
164
157
- self.embed_in = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
165
+ self.embed_in = VocabParallelEmbedding(config.vocab_size,
166
+ config.hidden_size,
167
perform_initialization=False)
159
- self.layers = nn.ModuleList([GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
160
- self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
168
+ self.layers = nn.ModuleList(
169
+ [GPTNeoXLayer(config) for _ in range(config.num_hidden_layers)])
170
+ self.final_layer_norm = nn.LayerNorm(config.hidden_size,
171
+ eps=config.layer_norm_eps)
172
173
def forward(
174
self,
@@ -191,8 +202,10 @@ class GPTNeoXForCausalLM(nn.Module):
202
super().__init__()
203
self.config = config
204
self.gpt_neox = GPTNeoXModel(config)
194
- self.embed_out = ColumnParallelLinear(config.hidden_size, config.vocab_size,
195
- bias=False, gather_output=False,
205
+ self.embed_out = ColumnParallelLinear(config.hidden_size,
206
+ config.vocab_size,
207
+ bias=False,
208
+ gather_output=False,
209
perform_initialization=False)
210
self.sampler = Sampler(config.vocab_size)
211
@@ -204,24 +217,28 @@ class GPTNeoXForCausalLM(nn.Module):
217
input_metadata: InputMetadata,
218
cache_events: Optional[List[torch.cuda.Event]],
219
) -> Dict[int, SequenceOutputs]:
207
- hidden_states = self.gpt_neox(
208
- input_ids, positions, kv_caches, input_metadata, cache_events)
209
- next_tokens = self.sampler(
210
- self.embed_out.weight, hidden_states, input_metadata)
220
+ hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
221
+ input_metadata, cache_events)
222
+ next_tokens = self.sampler(self.embed_out.weight, hidden_states,
223
+ input_metadata)
224
return next_tokens
225
213
- _column_parallel_weights = ["embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight", "dense_h_to_4h.bias"]
226
+ _column_parallel_weights = [
227
+ "embed_in.weight", "embed_out.weight", "dense_h_to_4h.weight",
228
+ "dense_h_to_4h.bias"
229
+ ]
230
_row_parallel_weights = ["dense.weight", "dense_4h_to_h.weight"]
231
216
- def load_weights(self, model_name_or_path: str,
232
+ def load_weights(self,
233
+ model_name_or_path: str,
234
cache_dir: Optional[str] = None,
235
use_np_cache: bool = False):
236
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
237
state_dict = self.state_dict()
238
for name, loaded_weight in hf_model_weights_iterator(
222
- model_name_or_path, cache_dir, use_np_cache):
239
+ model_name_or_path, cache_dir, use_np_cache):
240
if ("attention.bias" in name or "attention.masked_bias" in name
224
- or "rotary_emb.inv_freq" in name):
241
+ or "rotary_emb.inv_freq" in name):
242
continue
243
param = state_dict[name]
244
if "query_key_value" in name:
@@ -230,17 +247,19 @@ class GPTNeoXForCausalLM(nn.Module):
247
# required shape is [3 * num_heads * head_size, hidden_size].
248
# Thus, we need weight conversion.
249
shard_size = param.shape[0]
233
- loaded_weight = loaded_weight[shard_size * tensor_model_parallel_rank
234
- :shard_size * (tensor_model_parallel_rank + 1)]
250
+ loaded_weight = loaded_weight[
251
+ shard_size * tensor_model_parallel_rank:shard_size *
252
+ (tensor_model_parallel_rank + 1)]
253
254
num_heads = self.config.num_attention_heads
255
hidden_size = self.config.hidden_size
256
head_size = hidden_size // num_heads
239
- if 'query_key_value.weight' in name:
240
- loaded_weight = loaded_weight.view(-1, 3, head_size, hidden_size)
257
+ if "query_key_value.weight" in name:
258
+ loaded_weight = loaded_weight.view(-1, 3, head_size,
259
+ hidden_size)
260
loaded_weight = loaded_weight.transpose(0, 1)
261
loaded_weight = loaded_weight.reshape(-1, hidden_size)
243
- elif 'query_key_value.bias' in name:
262
+ elif "query_key_value.bias" in name:
263
loaded_weight = loaded_weight.view(-1, 3, head_size)
264
loaded_weight = loaded_weight.transpose(0, 1)
265
loaded_weight = loaded_weight.reshape(-1)
vllm/model_executor/models/llama.py
+57
-39
@@ -1,5 +1,6 @@
1
# coding=utf-8
2
-# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
# Copyright 2023 The vLLM team.
5
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
#
@@ -30,7 +31,6 @@ import torch
31
from torch import nn
32
from transformers import LlamaConfig
33
33
-from vllm.sequence import SequenceOutputs
34
from vllm.model_executor.input_metadata import InputMetadata
35
from vllm.model_executor.layers.activation import SiluAndMul
36
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -56,15 +56,19 @@ class LlamaMLP(nn.Module):
56
hidden_act: str,
57
):
58
super().__init__()
59
- self.gate_up_proj = ColumnParallelLinear(hidden_size, 2 * intermediate_size,
60
- bias=False, gather_output=False,
59
+ self.gate_up_proj = ColumnParallelLinear(hidden_size,
60
+ 2 * intermediate_size,
61
+ bias=False,
62
+ gather_output=False,
63
perform_initialization=False)
62
- self.down_proj = RowParallelLinear(intermediate_size, hidden_size,
63
- bias=False, input_is_parallel=True,
64
+ self.down_proj = RowParallelLinear(intermediate_size,
65
+ hidden_size,
66
+ bias=False,
67
+ input_is_parallel=True,
68
perform_initialization=False)
65
- if hidden_act != 'silu':
66
- raise ValueError(f'Unsupported activation: {hidden_act}. '
67
- 'Only silu is supported for now.')
69
+ if hidden_act != "silu":
70
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
71
+ "Only silu is supported for now.")
72
self.act_fn = SiluAndMul()
73
74
def forward(self, x):
@@ -83,12 +87,14 @@ class LlamaAttention(nn.Module):
87
):
88
super().__init__()
89
self.hidden_size = hidden_size
86
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
90
+ tensor_model_parallel_world_size = (
91
+ get_tensor_model_parallel_world_size())
92
self.total_num_heads = num_heads
93
assert self.total_num_heads % tensor_model_parallel_world_size == 0
89
- self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
94
+ self.num_heads = (self.total_num_heads //
95
+ tensor_model_parallel_world_size)
96
self.head_dim = hidden_size // self.total_num_heads
91
- self.scaling = self.head_dim ** -0.5
97
+ self.scaling = self.head_dim**-0.5
98
99
self.qkv_proj = ColumnParallelLinear(
100
hidden_size,
@@ -104,8 +110,10 @@ class LlamaAttention(nn.Module):
110
input_is_parallel=True,
111
perform_initialization=False,
112
)
107
- self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_dim,
108
- self.scaling, rotary_dim=self.head_dim)
113
+ self.attn = PagedAttentionWithRoPE(self.num_heads,
114
+ self.head_dim,
115
+ self.scaling,
116
+ rotary_dim=self.head_dim)
117
118
def forward(
119
self,
@@ -118,8 +126,8 @@ class LlamaAttention(nn.Module):
126
qkv, _ = self.qkv_proj(hidden_states)
127
q, k, v = qkv.chunk(chunks=3, dim=-1)
128
k_cache, v_cache = kv_cache
121
- attn_output = self.attn(
122
- positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
129
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
130
+ input_metadata, cache_event)
131
output, _ = self.o_proj(attn_output)
132
return output
133
@@ -138,8 +146,10 @@ class LlamaDecoderLayer(nn.Module):
146
intermediate_size=config.intermediate_size,
147
hidden_act=config.hidden_act,
148
)
141
- self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
142
- self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
149
+ self.input_layernorm = RMSNorm(config.hidden_size,
150
+ eps=config.rms_norm_eps)
151
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
152
+ eps=config.rms_norm_eps)
153
154
def forward(
155
self,
@@ -177,9 +187,13 @@ class LlamaModel(nn.Module):
187
self.padding_idx = config.pad_token_id
188
self.vocab_size = config.vocab_size
189
180
- self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
181
- perform_initialization=False)
182
- self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
190
+ self.embed_tokens = VocabParallelEmbedding(
191
+ config.vocab_size,
192
+ config.hidden_size,
193
+ perform_initialization=False)
194
+ self.layers = nn.ModuleList([
195
+ LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)
196
+ ])
197
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
198
199
def forward(
@@ -209,6 +223,7 @@ class LlamaModel(nn.Module):
223
224
225
class LlamaForCausalLM(nn.Module):
226
+
227
def __init__(self, config):
228
super().__init__()
229
self.config = config
@@ -228,39 +243,42 @@ class LlamaForCausalLM(nn.Module):
243
input_metadata: InputMetadata,
244
cache_events: Optional[List[torch.cuda.Event]],
245
) -> Dict[int, SequenceOutputs]:
231
- hidden_states = self.model(
232
- input_ids, positions, kv_caches, input_metadata, cache_events)
233
- next_tokens = self.sampler(
234
- self.lm_head.weight, hidden_states, input_metadata)
246
+ hidden_states = self.model(input_ids, positions, kv_caches,
247
+ input_metadata, cache_events)
248
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
249
+ input_metadata)
250
return next_tokens
251
237
- _column_parallel_weights = ["embed_tokens.weight", "lm_head.weight",
238
- "qkv_proj.weight", "gate_proj.weight",
239
- "up_proj.weight"]
252
+ _column_parallel_weights = [
253
+ "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
254
+ "gate_proj.weight", "up_proj.weight"
255
+ ]
256
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
257
242
- def load_weights(self, model_name_or_path: str,
258
+ def load_weights(self,
259
+ model_name_or_path: str,
260
cache_dir: Optional[str] = None,
261
use_np_cache: bool = False):
262
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
263
state_dict = self.state_dict()
264
265
for name, loaded_weight in hf_model_weights_iterator(
249
- model_name_or_path, cache_dir, use_np_cache):
266
+ model_name_or_path, cache_dir, use_np_cache):
267
if "rotary_emb.inv_freq" in name:
268
continue
269
270
is_attention_weight = False
254
- for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
271
+ for stride_id, att_weight_name in enumerate(
272
+ ["q_proj", "k_proj", "v_proj"]):
273
if att_weight_name not in name:
274
continue
275
param = state_dict[name.replace(att_weight_name, "qkv_proj")]
276
shard_size = param.shape[0] // 3
277
loaded_weight = loaded_weight[
260
- shard_size * tensor_model_parallel_rank
261
- :shard_size * (tensor_model_parallel_rank + 1)]
262
- param_slice = param.data[shard_size * stride_id
263
- :shard_size * (stride_id + 1)]
278
+ shard_size * tensor_model_parallel_rank:shard_size *
279
+ (tensor_model_parallel_rank + 1)]
280
+ param_slice = param.data[shard_size * stride_id:shard_size *
281
+ (stride_id + 1)]
282
assert param_slice.shape == loaded_weight.shape
283
param_slice.copy_(loaded_weight)
284
is_attention_weight = True
@@ -275,10 +293,10 @@ class LlamaForCausalLM(nn.Module):
293
param = state_dict[name.replace(weight_name, "gate_up_proj")]
294
shard_size = param.shape[0] // 2
295
loaded_weight = loaded_weight[
278
- shard_size * tensor_model_parallel_rank
279
- :shard_size * (tensor_model_parallel_rank + 1)]
280
- param_slice = param.data[shard_size * stride_id
281
- :shard_size * (stride_id + 1)]
296
+ shard_size * tensor_model_parallel_rank:shard_size *
297
+ (tensor_model_parallel_rank + 1)]
298
+ param_slice = param.data[shard_size * stride_id:shard_size *
299
+ (stride_id + 1)]
300
assert param_slice.shape == loaded_weight.shape
301
param_slice.copy_(loaded_weight)
302
is_gate_up_weight = True
vllm/model_executor/models/opt.py
+69
-46
@@ -1,7 +1,9 @@
1
# coding=utf-8
2
-# Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/opt/modeling_opt.py
4
# Copyright 2023 The vLLM team.
4
-# Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved.
5
+# Copyright 2022 The Fairseq Authors and The HuggingFace Inc. team. All rights
6
+# reserved.
7
#
8
# Licensed under the Apache License, Version 2.0 (the "License");
9
# you may not use this file except in compliance with the License.
@@ -43,8 +45,9 @@ KVCache = Tuple[torch.Tensor, torch.Tensor]
45
class OPTLearnedPositionalEmbedding(nn.Embedding):
46
47
def __init__(self, num_embeddings: int, embedding_dim: int):
46
- # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2
47
- # and adjust num_embeddings appropriately. Other models don't have this hack
48
+ # OPT is set up so that if padding_idx is specified then offset the
49
+ # embedding ids by 2 and adjust num_embeddings appropriately. Other
50
+ # models don't have this hack
51
self.offset = 2
52
super().__init__(num_embeddings + self.offset, embedding_dim)
53
@@ -62,20 +65,26 @@ class OPTAttention(nn.Module):
65
) -> None:
66
super().__init__()
67
self.embed_dim = embed_dim
65
- tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
68
+ tensor_model_parallel_world_size = (
69
+ get_tensor_model_parallel_world_size())
70
total_num_heads = num_heads
71
assert num_heads % tensor_model_parallel_world_size == 0
72
self.num_heads = total_num_heads // tensor_model_parallel_world_size
73
self.head_dim = embed_dim // total_num_heads
70
- self.scaling = self.head_dim ** -0.5
74
+ self.scaling = self.head_dim**-0.5
75
72
- self.qkv_proj = ColumnParallelLinear(embed_dim, 3 * embed_dim, bias=bias,
76
+ self.qkv_proj = ColumnParallelLinear(embed_dim,
77
+ 3 * embed_dim,
78
+ bias=bias,
79
gather_output=False,
80
perform_initialization=False)
75
- self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
81
+ self.out_proj = RowParallelLinear(embed_dim,
82
+ embed_dim,
83
+ bias=bias,
84
input_is_parallel=True,
85
perform_initialization=False)
78
- self.attn = PagedAttention(self.num_heads, self.head_dim,
86
+ self.attn = PagedAttention(self.num_heads,
87
+ self.head_dim,
88
scale=self.scaling)
89
90
def forward(
@@ -88,8 +97,8 @@ class OPTAttention(nn.Module):
97
qkv, _ = self.qkv_proj(hidden_states)
98
q, k, v = qkv.chunk(chunks=3, dim=-1)
99
key_cache, value_cache = kv_cache
91
- attn_output = self.attn(
92
- q, k, v, key_cache, value_cache, input_metadata, cache_event)
100
+ attn_output = self.attn(q, k, v, key_cache, value_cache,
101
+ input_metadata, cache_event)
102
output, _ = self.out_proj(attn_output)
103
return output
104
@@ -109,17 +118,21 @@ class OPTDecoderLayer(nn.Module):
118
self.activation_fn = get_act_fn(config.activation_function)
119
120
self.self_attn_layer_norm = nn.LayerNorm(
112
- self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
113
- self.fc1 = ColumnParallelLinear(self.embed_dim, config.ffn_dim,
121
+ self.embed_dim,
122
+ elementwise_affine=config.layer_norm_elementwise_affine)
123
+ self.fc1 = ColumnParallelLinear(self.embed_dim,
124
+ config.ffn_dim,
125
bias=config.enable_bias,
126
gather_output=False,
127
perform_initialization=False)
117
- self.fc2 = RowParallelLinear(config.ffn_dim, self.embed_dim,
128
+ self.fc2 = RowParallelLinear(config.ffn_dim,
129
+ self.embed_dim,
130
bias=config.enable_bias,
131
input_is_parallel=True,
132
perform_initialization=False)
133
self.final_layer_norm = nn.LayerNorm(
122
- self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine)
134
+ self.embed_dim,
135
+ elementwise_affine=config.layer_norm_elementwise_affine)
136
137
def forward(
138
self,
@@ -133,11 +146,10 @@ class OPTDecoderLayer(nn.Module):
146
# 125m, 1.7B, ..., 175B applies layer norm BEFORE attention
147
if self.do_layer_norm_before:
148
hidden_states = self.self_attn_layer_norm(hidden_states)
136
- hidden_states = self.self_attn(
137
- hidden_states=hidden_states,
138
- kv_cache=kv_cache,
139
- input_metadata=input_metadata,
140
- cache_event=cache_event)
149
+ hidden_states = self.self_attn(hidden_states=hidden_states,
150
+ kv_cache=kv_cache,
151
+ input_metadata=input_metadata,
152
+ cache_event=cache_event)
153
hidden_states = residual + hidden_states
154
# 350m applies layer norm AFTER attention
155
if not self.do_layer_norm_before:
@@ -167,35 +179,42 @@ class OPTDecoder(nn.Module):
179
self.max_target_positions = config.max_position_embeddings
180
self.vocab_size = config.vocab_size
181
170
- self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
171
- config.word_embed_proj_dim,
172
- perform_initialization=False)
182
+ self.embed_tokens = VocabParallelEmbedding(
183
+ config.vocab_size,
184
+ config.word_embed_proj_dim,
185
+ perform_initialization=False)
186
# Positional embeddings are replicated (not sharded).
187
self.embed_positions = OPTLearnedPositionalEmbedding(
188
config.max_position_embeddings, config.hidden_size)
189
190
# Project out & in will be replicated if they exist.
191
if config.word_embed_proj_dim != config.hidden_size:
179
- self.project_out = nn.Linear(config.hidden_size, config.word_embed_proj_dim, bias=False)
192
+ self.project_out = nn.Linear(config.hidden_size,
193
+ config.word_embed_proj_dim,
194
+ bias=False)
195
else:
196
self.project_out = None
197
198
if config.word_embed_proj_dim != config.hidden_size:
184
- self.project_in = nn.Linear(config.word_embed_proj_dim, config.hidden_size, bias=False)
199
+ self.project_in = nn.Linear(config.word_embed_proj_dim,
200
+ config.hidden_size,
201
+ bias=False)
202
else:
203
self.project_in = None
204
188
- # Note that the only purpose of `config._remove_final_layer_norm` is to keep backward compatibility
189
- # with checkpoints that have been fine-tuned before transformers v4.20.1
205
+ # Note that the only purpose of `config._remove_final_layer_norm` is to
206
+ # keep backward compatibility with checkpoints that have been fine-tuned
207
+ # before transformers v4.20.1
208
# see https://github.com/facebookresearch/metaseq/pull/164
209
if config.do_layer_norm_before and not config._remove_final_layer_norm:
210
self.final_layer_norm = nn.LayerNorm(
193
- config.hidden_size, elementwise_affine=config.layer_norm_elementwise_affine
194
- )
211
+ config.hidden_size,
212
+ elementwise_affine=config.layer_norm_elementwise_affine)
213
else:
214
self.final_layer_norm = None
215
198
- self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
216
+ self.layers = nn.ModuleList(
217
+ [OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)])
218
219
def forward(
220
self,
@@ -217,8 +236,8 @@ class OPTDecoder(nn.Module):
236
else:
237
cache_event = cache_events[i]
238
layer = self.layers[i]
220
- hidden_states = layer(
221
- hidden_states, kv_caches[i], input_metadata, cache_event)
239
+ hidden_states = layer(hidden_states, kv_caches[i], input_metadata,
240
+ cache_event)
241
242
if self.final_layer_norm is not None:
243
hidden_states = self.final_layer_norm(hidden_states)
@@ -241,8 +260,8 @@ class OPTModel(nn.Module):
260
input_metadata: InputMetadata,
261
cache_events: Optional[List[torch.cuda.Event]],
262
) -> torch.Tensor:
244
- return self.decoder(
245
- input_ids, positions, kv_caches, input_metadata, cache_events)
263
+ return self.decoder(input_ids, positions, kv_caches, input_metadata,
264
+ cache_events)
265
266
267
class OPTForCausalLM(nn.Module):
@@ -264,23 +283,26 @@ class OPTForCausalLM(nn.Module):
283
input_metadata: InputMetadata,
284
cache_events: Optional[List[torch.cuda.Event]],
285
) -> Dict[int, SequenceOutputs]:
267
- hidden_states = self.model(
268
- input_ids, positions, kv_caches, input_metadata, cache_events)
269
- next_tokens = self.sampler(
270
- self.lm_head_weight, hidden_states, input_metadata)
286
+ hidden_states = self.model(input_ids, positions, kv_caches,
287
+ input_metadata, cache_events)
288
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
289
+ input_metadata)
290
return next_tokens
291
273
- _column_parallel_weights = ["embed_tokens.weight", "fc1.weight", "fc1.bias"]
292
+ _column_parallel_weights = [
293
+ "embed_tokens.weight", "fc1.weight", "fc1.bias"
294
+ ]
295
_row_parallel_weights = ["out_proj.weight", "fc2.weight"]
296
276
- def load_weights(self, model_name_or_path: str,
297
+ def load_weights(self,
298
+ model_name_or_path: str,
299
cache_dir: Optional[str] = None,
300
use_np_cache: bool = False):
301
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
302
state_dict = self.state_dict()
303
304
for name, loaded_weight in hf_model_weights_iterator(
283
- model_name_or_path, cache_dir, use_np_cache):
305
+ model_name_or_path, cache_dir, use_np_cache):
306
if "lm_head.weight" in name:
307
continue
308
@@ -288,16 +310,17 @@ class OPTForCausalLM(nn.Module):
310
name = "model." + name
311
312
is_attention_weight = False
291
- for stride_id, att_weight_name in enumerate(["q_proj", "k_proj", "v_proj"]):
313
+ for stride_id, att_weight_name in enumerate(
314
+ ["q_proj", "k_proj", "v_proj"]):
315
if att_weight_name not in name:
316
continue
317
param = state_dict[name.replace(att_weight_name, "qkv_proj")]
318
shard_size = param.shape[0] // 3
319
loaded_weight = loaded_weight[
297
- shard_size * tensor_model_parallel_rank
298
- :shard_size * (tensor_model_parallel_rank + 1)]
299
- param_slice = param.data[shard_size * stride_id
300
- :shard_size * (stride_id + 1)]
320
+ shard_size * tensor_model_parallel_rank:shard_size *
321
+ (tensor_model_parallel_rank + 1)]
322
+ param_slice = param.data[shard_size * stride_id:shard_size *
323
+ (stride_id + 1)]
324
assert param_slice.shape == loaded_weight.shape
325
param_slice.copy_(loaded_weight)
326
is_attention_weight = True
vllm/model_executor/weight_utils.py
+10
-11
@@ -44,9 +44,9 @@ def hf_model_weights_iterator(
44
if use_np_cache:
45
# Convert the model weights from torch tensors to numpy arrays for
46
# faster loading.
47
- np_folder = os.path.join(hf_folder, 'np')
47
+ np_folder = os.path.join(hf_folder, "np")
48
os.makedirs(np_folder, exist_ok=True)
49
- weight_names_file = os.path.join(np_folder, 'weight_names.json')
49
+ weight_names_file = os.path.join(np_folder, "weight_names.json")
50
with lock:
51
if not os.path.exists(weight_names_file):
52
weight_names = []
@@ -57,10 +57,10 @@ def hf_model_weights_iterator(
57
with open(param_path, "wb") as f:
58
np.save(f, param.cpu().detach().numpy())
59
weight_names.append(name)
60
- with open(weight_names_file, 'w') as f:
60
+ with open(weight_names_file, "w") as f:
61
json.dump(weight_names, f)
62
63
- with open(weight_names_file, 'r') as f:
63
+ with open(weight_names_file, "r") as f:
64
weight_names = json.load(f)
65
66
for name in weight_names:
@@ -86,17 +86,16 @@ def load_tensor_parallel_weights(
86
for p in column_parallel_weight_names:
87
if p in param_name:
88
shard_size = param.shape[0]
89
- loaded_weight = loaded_weight[
90
- shard_size * tensor_model_parallel_rank
91
- :shard_size * (tensor_model_parallel_rank + 1)]
89
+ start_idx = tensor_model_parallel_rank * shard_size
90
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
91
+ loaded_weight = loaded_weight[start_idx:end_idx]
92
break
93
for p in row_parallel_weight_names:
94
if p in param_name:
95
shard_size = param.shape[1]
96
- loaded_weight = loaded_weight[
97
- :,
98
- shard_size * tensor_model_parallel_rank
99
- :shard_size * (tensor_model_parallel_rank + 1)]
96
+ start_idx = tensor_model_parallel_rank * shard_size
97
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
98
+ loaded_weight = loaded_weight[:, start_idx:end_idx]
99
break
100
assert param.shape == loaded_weight.shape, (
101
f"{param_name} shape mismatch between model and checkpoint: "
vllm/outputs.py
+4
-2
@@ -55,6 +55,7 @@ class RequestOutput:
55
outputs: The output sequences of the request.
56
finished: Whether the whole request is finished.
57
"""
58
+
59
def __init__(
60
self,
61
request_id: str,
@@ -75,8 +76,9 @@ class RequestOutput:
76
n = seq_group.sampling_params.n
77
seqs = seq_group.get_seqs()
78
assert n <= len(seqs)
78
- sorted_seqs = sorted(
79
- seqs, key=lambda seq: seq.get_cumulative_logprob(), reverse=True)
79
+ sorted_seqs = sorted(seqs,
80
+ key=lambda seq: seq.get_cumulative_logprob(),
81
+ reverse=True)
82
top_n_seqs = sorted_seqs[:n]
83
84
# Create the outputs.
vllm/sampling_params.py
+8
-2
@@ -3,6 +3,7 @@ from typing import List, Optional, Union
3
4
_SAMPLING_EPS = 1e-5
5
6
+
7
class SamplingParams:
8
"""Sampling parameters for text generation.
9
@@ -51,7 +52,7 @@ class SamplingParams:
52
top_p: float = 1.0,
53
top_k: int = -1,
54
use_beam_search: bool = False,
54
- stop: Union[str, List[str]] = [],
55
+ stop: Union[None, str, List[str]] = None,
56
ignore_eos: bool = False,
57
max_tokens: int = 16,
58
logprobs: Optional[int] = None,
@@ -64,7 +65,12 @@ class SamplingParams:
65
self.top_p = top_p
66
self.top_k = top_k
67
self.use_beam_search = use_beam_search
67
- self.stop = [stop] if isinstance(stop, str) else list(stop)
68
+ if stop is None:
69
+ self.stop = []
70
+ elif isinstance(stop, str):
71
+ self.stop = [stop]
72
+ else:
73
+ self.stop = list(stop)
74
self.ignore_eos = ignore_eos
75
self.max_tokens = max_tokens
76
self.logprobs = logprobs
vllm/sequence.py
+70
-19
@@ -1,3 +1,4 @@
1
+"""Sequence and its related classes."""
2
import copy
3
import enum
4
from typing import Dict, List, Optional, Union
@@ -7,6 +8,7 @@ from vllm.sampling_params import SamplingParams
8
9
10
class SequenceStatus(enum.Enum):
11
+ """Status of a sequence."""
12
WAITING = enum.auto()
13
RUNNING = enum.auto()
14
SWAPPED = enum.auto()
@@ -21,7 +23,7 @@ class SequenceStatus(enum.Enum):
23
SequenceStatus.FINISHED_STOPPED,
24
SequenceStatus.FINISHED_LENGTH_CAPPED,
25
SequenceStatus.FINISHED_ABORTED,
24
- SequenceStatus.FINISHED_IGNORED
26
+ SequenceStatus.FINISHED_IGNORED,
27
]
28
29
@staticmethod
@@ -40,6 +42,17 @@ class SequenceStatus(enum.Enum):
42
43
44
class SequenceData:
45
+ """Data associated with a sequence.
46
+
47
+
48
+ Args:
49
+ prompt_token_ids: The token IDs of the prompt.
50
+
51
+ Attributes:
52
+ prompt_token_ids: The token IDs of the prompt.
53
+ output_token_ids: The token IDs of the output.
54
+ cumulative_logprob: The cumulative log probability of the output.
55
+ """
56
57
def __init__(
58
self,
@@ -75,6 +88,15 @@ class SequenceData:
88
89
90
class Sequence:
91
+ """Stores the data, status, and block information of a sequence.
92
+
93
+ Args:
94
+ seq_id: The ID of the sequence.
95
+ prompt: The prompt of the sequence.
96
+ prompt_token_ids: The token IDs of the prompt.
97
+ block_size: The block size of the sequence. Should be the same as the
98
+ block size used by the block manager and cache engine.
99
+ """
100
101
def __init__(
102
self,
@@ -149,19 +171,27 @@ class Sequence:
171
def is_finished(self) -> bool:
172
return SequenceStatus.is_finished(self.status)
173
152
- def fork(self, child_seq: 'Sequence') -> None:
153
- child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
174
+ def fork(self, child_seq: "Sequence") -> None:
175
+ child_seq.logical_token_blocks = copy.deepcopy(
176
+ self.logical_token_blocks)
177
child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
178
child_seq.data = copy.deepcopy(self.data)
156
- return None
179
180
def __repr__(self) -> str:
159
- return (f'Sequence(seq_id={self.seq_id}, '
160
- f'status={self.status.name}, '
161
- f'num_blocks={len(self.logical_token_blocks)})')
181
+ return (f"Sequence(seq_id={self.seq_id}, "
182
+ f"status={self.status.name}, "
183
+ f"num_blocks={len(self.logical_token_blocks)})")
184
185
186
class SequenceGroup:
187
+ """A group of sequences that are generated from the same prompt.
188
+
189
+ Args:
190
+ request_id: The ID of the request.
191
+ seqs: The list of sequences.
192
+ sampling_params: The sampling parameters used to generate the outputs.
193
+ arrival_time: The arrival time of the request.
194
+ """
195
196
def __init__(
197
self,
@@ -191,7 +221,7 @@ class SequenceGroup:
221
for seq in self.seqs:
222
if seq.seq_id == seq_id:
223
return seq
194
- raise ValueError(f'Sequence {seq_id} not found.')
224
+ raise ValueError(f"Sequence {seq_id} not found.")
225
226
def is_finished(self) -> bool:
227
return all(seq.is_finished() for seq in self.seqs)
@@ -203,14 +233,25 @@ class SequenceGroup:
233
234
235
class SequenceGroupMetadata:
236
+ """Metadata for a sequence group. Used to create `InputMetadata`.
237
+
238
+
239
+ Args:
240
+ request_id: The ID of the request.
241
+ is_prompt: Whether the request is at prompt stage.
242
+ seq_data: The sequence data. (Seq id -> sequence data)
243
+ sampling_params: The sampling parameters used to generate the outputs.
244
+ block_tables: The block tables. (Seq id -> list of physical block
245
+ numbers)
246
+ """
247
248
def __init__(
249
self,
250
request_id: str,
251
is_prompt: bool,
211
- seq_data: Dict[int, SequenceData], # Seq id -> sequence data.
252
+ seq_data: Dict[int, SequenceData],
253
sampling_params: SamplingParams,
213
- block_tables: Dict[int, List[int]], # Seq id -> list of physical block numbers.
254
+ block_tables: Dict[int, List[int]],
255
) -> None:
256
self.request_id = request_id
257
self.is_prompt = is_prompt
@@ -220,13 +261,23 @@ class SequenceGroupMetadata:
261
262
263
class SequenceOutputs:
264
+ """The model output associated with a sequence.
265
+
266
+ Args:
267
+ seq_id: The ID of the sequence.
268
+ parent_seq_id: The ID of the parent sequence (for forking in beam
269
+ search).
270
+ output_token: The output token ID.
271
+ logprobs: The logprobs of the output token.
272
+ (Token id -> logP(x_i+1 | x_0, ..., x_i))
273
+ """
274
275
def __init__(
276
self,
277
seq_id: int,
278
parent_seq_id: int,
279
output_token: int,
229
- logprobs: Dict[int, float], # Token id -> logP(x_i+1 | x_0, ..., x_i).
280
+ logprobs: Dict[int, float],
281
) -> None:
282
self.seq_id = seq_id
283
self.parent_seq_id = parent_seq_id
@@ -234,15 +285,15 @@ class SequenceOutputs:
285
self.logprobs = logprobs
286
287
def __repr__(self) -> str:
237
- return (f'SequenceOutputs(seq_id={self.seq_id}, '
238
- f'parent_seq_id={self.parent_seq_id}, '
239
- f'output_token={self.output_token}), '
240
- f'logprobs={self.logprobs}')
288
+ return (f"SequenceOutputs(seq_id={self.seq_id}, "
289
+ f"parent_seq_id={self.parent_seq_id}, "
290
+ f"output_token={self.output_token}), "
291
+ f"logprobs={self.logprobs}")
292
293
def __eq__(self, other: object) -> bool:
294
if not isinstance(other, SequenceOutputs):
295
return NotImplemented
245
- return (self.seq_id == other.seq_id and
246
- self.parent_seq_id == other.parent_seq_id and
247
- self.output_token == other.output_token and
248
- self.logprobs == other.logprobs)
296
+ return (self.seq_id == other.seq_id
297
+ and self.parent_seq_id == other.parent_seq_id
298
+ and self.output_token == other.output_token
299
+ and self.logprobs == other.logprobs)
vllm/transformers_utils/tokenizer.py
+3
-2
@@ -13,8 +13,8 @@ _FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
13
14
def get_tokenizer(
15
tokenizer_name: str,
16
- tokenizer_mode: str = "auto",
16
*args,
17
+ tokenizer_mode: str = "auto",
18
**kwargs,
19
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
20
"""Gets a tokenizer for the given model name via Huggingface."""
@@ -73,7 +73,8 @@ def detokenize_incrementally(
73
output_text = tokenizer.convert_tokens_to_string(output_tokens)
74
return new_token, output_text
75
76
- # Adapted from https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
76
+ # Adapted from
77
+ # https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/tokenization_utils.py#L921
78
# NOTE(woosuk): The following code is slow because it runs a for loop over
79
# the output_tokens. In Python, running a for loop over a list can be slow
80
# even when the loop body is very simple.
vllm/utils.py
+3
-2
@@ -17,9 +17,9 @@ class Counter:
17
self.counter = start
18
19
def __next__(self) -> int:
20
- id = self.counter
20
+ i = self.counter
21
self.counter += 1
22
- return id
22
+ return i
23
24
def reset(self) -> None:
25
self.counter = 0
@@ -38,6 +38,7 @@ def get_cpu_memory() -> int:
38
def random_uuid() -> str:
39
return str(uuid.uuid4().hex)
40
41
+
42
def in_wsl() -> bool:
43
# Reference: https://github.com/microsoft/WSL/issues/4071
44
return "microsoft" in " ".join(uname()).lower()
vllm/worker/cache_engine.py
+5
-6
@@ -93,8 +93,8 @@ class CacheEngine:
93
if not pin_memory:
94
# Pinning memory in WSL is not supported.
95
# https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations-for-linux-cuda-applications
96
- logger.warn("Using 'pin_memory=False' as WSL is detected. "
97
- "This may slow down the performance.")
96
+ logger.warning("Using 'pin_memory=False' as WSL is detected. "
97
+ "This may slow down the performance.")
98
for _ in range(self.num_layers):
99
key_blocks = torch.empty(
100
size=(self.num_cpu_blocks, *key_block_shape),
@@ -120,11 +120,10 @@ class CacheEngine:
120
src_key_cache, src_value_cache = src[i]
121
dst_key_cache, dst_value_cache = dst[i]
122
# Copy the key blocks.
123
- cache_ops.swap_blocks(
124
- src_key_cache, dst_key_cache, src_to_dst)
123
+ cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
124
# Copy the value blocks.
126
- cache_ops.swap_blocks(
127
- src_value_cache, dst_value_cache, src_to_dst)
125
+ cache_ops.swap_blocks(src_value_cache, dst_value_cache,
126
+ src_to_dst)
127
event = self.events[i]
128
event.record(stream=self.cache_stream)
129
vllm/worker/worker.py
+13
-10
@@ -73,8 +73,8 @@ class Worker:
73
# number of tokens equal to max_num_batched_tokens.
74
75
# Enable top-k sampling to reflect the accurate memory usage.
76
- sampling_params = SamplingParams(top_p=0.99,
77
- top_k=self.model.config.vocab_size - 1)
76
+ vocab_size = self.model.config.vocab_size
77
+ sampling_params = SamplingParams(top_p=0.99, top_k=vocab_size - 1)
78
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
79
max_num_seqs = self.scheduler_config.max_num_seqs
80
seqs = []
@@ -91,7 +91,8 @@ class Worker:
91
)
92
seqs.append(seq)
93
94
- input_tokens, input_positions, input_metadata = self._prepare_inputs(seqs)
94
+ input_tokens, input_positions, input_metadata = self._prepare_inputs(
95
+ seqs)
96
97
# Execute the model.
98
num_layers = self.model_config.get_num_layers(self.parallel_config)
@@ -110,8 +111,9 @@ class Worker:
111
total_gpu_memory = get_gpu_memory()
112
cache_block_size = CacheEngine.get_cache_block_size(
113
block_size, self.model_config, self.parallel_config)
113
- num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
114
- - peak_memory) // cache_block_size)
114
+ num_gpu_blocks = int(
115
+ (total_gpu_memory * gpu_memory_utilization - peak_memory) //
116
+ cache_block_size)
117
num_cpu_blocks = int(cpu_swap_space // cache_block_size)
118
num_gpu_blocks = max(num_gpu_blocks, 0)
119
num_cpu_blocks = max(num_cpu_blocks, 0)
@@ -125,8 +127,8 @@ class Worker:
127
def init_cache_engine(self, cache_config: CacheConfig) -> None:
128
self.cache_config = cache_config
129
self.block_size = cache_config.block_size
128
- self.cache_engine = CacheEngine(
129
- self.cache_config, self.model_config, self.parallel_config)
130
+ self.cache_engine = CacheEngine(self.cache_config, self.model_config,
131
+ self.parallel_config)
132
self.cache_events = self.cache_engine.events
133
self.gpu_cache = self.cache_engine.gpu_cache
134
@@ -202,8 +204,8 @@ class Worker:
204
generation_block_tables.append(block_table)
205
206
max_context_len = max(max_context_len, context_len)
205
- max_num_blocks_per_seq = max(
206
- max_num_blocks_per_seq, len(block_table))
207
+ max_num_blocks_per_seq = max(max_num_blocks_per_seq,
208
+ len(block_table))
209
context_lens.append(context_len)
210
211
block_number = block_table[position // self.block_size]
@@ -223,7 +225,8 @@ class Worker:
225
context_lens_tensor = torch.cuda.IntTensor(context_lens)
226
padded_block_tables = [
227
_pad_to_max(block_table, max_num_blocks_per_seq)
226
- for block_table in generation_block_tables]
228
+ for block_table in generation_block_tables
229
+ ]
230
block_tables_tensor = torch.cuda.IntTensor(padded_block_tables)
231
232
seq_data: Dict[int, SequenceData] = {}