A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[ci/build] Pin CI dependencies version with pip-compile (#9810)
Signed-off-by: kevin <kevin@anyscale.com>
Kevin H. Luu committed
Oct 29, 2024 at 17:34 UTC
62fac4b9aab3c05124d83fcd71db5732774b17d8
4 files changed
+608
-42
Dockerfile.rocm
+2
@@ -121,6 +121,8 @@ ARG GIT_REPO_CHECK=0
121
RUN --mount=type=bind,source=.git,target=.git \
122
if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
123
124
+RUN python3 -m pip install --upgrade pip
125
+
126
# Package upgrades for useful functionality or to avoid dependency issues
127
RUN --mount=type=cache,target=/root/.cache/pip \
128
python3 -m pip install --upgrade numba scipy huggingface-hub[cli] pytest-shard
requirements-build.txt
+9
-9
@@ -1,9 +1,9 @@
1
-# Should be mirrored in pyproject.toml
2
-cmake>=3.26
3
-ninja
4
-packaging
5
-setuptools>=61
6
-setuptools-scm>=8
7
-torch==2.5.0
8
-wheel
9
-jinja2
1
+# Should be mirrored in pyproject.toml
2
+cmake>=3.26
3
+ninja
4
+packaging
5
+setuptools>=61
6
+setuptools-scm>=8
7
+torch==2.5.0
8
+wheel
9
+jinja2
requirements-test.in
new
+37
@@ -0,0 +1,37 @@
1
+# testing
2
+pytest
3
+tensorizer>=2.9.0
4
+pytest-forked
5
+pytest-asyncio
6
+pytest-rerunfailures
7
+pytest-shard
8
+
9
+# testing utils
10
+awscli
11
+einops # required for MPT, qwen-vl and Mamba
12
+httpx
13
+librosa # required for audio tests
14
+opencv-python # required for video tests
15
+peft
16
+requests
17
+ray[adag]==2.35
18
+sentence-transformers # required for embedding
19
+soundfile # required for audio test
20
+timm # required for internvl test
21
+torch==2.5.0
22
+transformers_stream_generator # required for qwen-vl test
23
+matplotlib # required for qwen-vl test
24
+datamodel_code_generator # required for minicpm3 test
25
+lm-eval[api]==0.4.4 # required for model evaluation test
26
+
27
+# TODO: Add this after fully implementing llava(mantis)
28
+# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
29
+
30
+# Benchmarking
31
+aiohttp
32
+
33
+# quantization
34
+bitsandbytes>=0.44.0
35
+buildkite-test-collector==0.1.8
36
+
37
+numpy < 2.0.0
requirements-test.txt
+560
-33
@@ -1,34 +1,561 @@
1
-# testing
2
-pytest
3
-tensorizer>=2.9.0
4
-pytest-forked
5
-pytest-asyncio
6
-pytest-rerunfailures
7
-pytest-shard
8
-
9
-# testing utils
10
-awscli
11
-einops # required for MPT, qwen-vl and Mamba
12
-httpx
13
-librosa # required for audio tests
14
-opencv-python # required for video tests
15
-peft
16
-requests
17
-ray[adag]==2.35
18
-sentence-transformers # required for embedding
19
-soundfile # required for audio test
20
-timm # required for internvl test
21
-transformers_stream_generator # required for qwen-vl test
22
-matplotlib # required for qwen-vl test
23
-datamodel_code_generator # required for minicpm3 test
24
-lm-eval[api]==0.4.4 # required for model evaluation test
25
-
26
-# TODO: Add this after fully implementing llava(mantis)
27
-# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
28
-
29
-# Benchmarking
30
-aiohttp
31
-
32
-# quantization
33
-bitsandbytes>=0.44.0
1
+#
2
+# This file is autogenerated by pip-compile with Python 3.12
3
+# by the following command:
4
+#
5
+# pip-compile --output-file=requirements-test.txt requirements-test.in
6
+#
7
+absl-py==2.1.0
8
+ # via rouge-score
9
+accelerate==1.0.1
10
+ # via
11
+ # lm-eval
12
+ # peft
13
+aiohappyeyeballs==2.4.3
14
+ # via aiohttp
15
+aiohttp==3.10.10
16
+ # via
17
+ # -r requirements-test.in
18
+ # datasets
19
+ # fsspec
20
+ # lm-eval
21
+aiosignal==1.3.1
22
+ # via
23
+ # aiohttp
24
+ # ray
25
+annotated-types==0.7.0
26
+ # via pydantic
27
+anyio==4.6.2.post1
28
+ # via httpx
29
+argcomplete==3.5.1
30
+ # via datamodel-code-generator
31
+attrs==24.2.0
32
+ # via
33
+ # aiohttp
34
+ # jsonlines
35
+ # jsonschema
36
+ # referencing
37
+audioread==3.0.1
38
+ # via librosa
39
+awscli==1.35.16
40
+ # via -r requirements-test.in
41
+bitsandbytes==0.44.1
42
+ # via -r requirements-test.in
43
+black==24.10.0
44
+ # via datamodel-code-generator
45
+boto3==1.35.50
46
+ # via tensorizer
47
+botocore==1.35.50
48
+ # via
49
+ # awscli
50
+ # boto3
51
+ # s3transfer
52
buildkite-test-collector==0.1.8
53
+ # via -r requirements-test.in
54
+certifi==2024.8.30
55
+ # via
56
+ # httpcore
57
+ # httpx
58
+ # requests
59
+cffi==1.17.1
60
+ # via soundfile
61
+chardet==5.2.0
62
+ # via mbstrdecoder
63
+charset-normalizer==3.4.0
64
+ # via requests
65
+click==8.1.7
66
+ # via
67
+ # black
68
+ # nltk
69
+ # ray
70
+colorama==0.4.6
71
+ # via
72
+ # awscli
73
+ # sacrebleu
74
+ # tqdm-multiprocess
75
+contourpy==1.3.0
76
+ # via matplotlib
77
+cupy-cuda12x==13.3.0
78
+ # via ray
79
+cycler==0.12.1
80
+ # via matplotlib
81
+datamodel-code-generator==0.26.2
82
+ # via -r requirements-test.in
83
+dataproperty==1.0.1
84
+ # via
85
+ # pytablewriter
86
+ # tabledata
87
+datasets==3.0.2
88
+ # via
89
+ # evaluate
90
+ # lm-eval
91
+decorator==5.1.1
92
+ # via librosa
93
+dill==0.3.8
94
+ # via
95
+ # datasets
96
+ # evaluate
97
+ # lm-eval
98
+ # multiprocess
99
+dnspython==2.7.0
100
+ # via email-validator
101
+docutils==0.16
102
+ # via awscli
103
+einops==0.8.0
104
+ # via -r requirements-test.in
105
+email-validator==2.2.0
106
+ # via pydantic
107
+evaluate==0.4.3
108
+ # via lm-eval
109
+fastrlock==0.8.2
110
+ # via cupy-cuda12x
111
+filelock==3.16.1
112
+ # via
113
+ # datasets
114
+ # huggingface-hub
115
+ # ray
116
+ # torch
117
+ # transformers
118
+ # triton
119
+fonttools==4.54.1
120
+ # via matplotlib
121
+frozenlist==1.5.0
122
+ # via
123
+ # aiohttp
124
+ # aiosignal
125
+ # ray
126
+fsspec[http]==2024.9.0
127
+ # via
128
+ # datasets
129
+ # evaluate
130
+ # huggingface-hub
131
+ # torch
132
+genson==1.3.0
133
+ # via datamodel-code-generator
134
+h11==0.14.0
135
+ # via httpcore
136
+hiredis==3.0.0
137
+ # via tensorizer
138
+httpcore==1.0.6
139
+ # via httpx
140
+httpx==0.27.2
141
+ # via -r requirements-test.in
142
+huggingface-hub==0.26.2
143
+ # via
144
+ # accelerate
145
+ # datasets
146
+ # evaluate
147
+ # peft
148
+ # sentence-transformers
149
+ # timm
150
+ # tokenizers
151
+ # transformers
152
+idna==3.10
153
+ # via
154
+ # anyio
155
+ # email-validator
156
+ # httpx
157
+ # requests
158
+ # yarl
159
+inflect==5.6.2
160
+ # via datamodel-code-generator
161
+iniconfig==2.0.0
162
+ # via pytest
163
+isort==5.13.2
164
+ # via datamodel-code-generator
165
+jinja2==3.1.4
166
+ # via
167
+ # datamodel-code-generator
168
+ # torch
169
+jmespath==1.0.1
170
+ # via
171
+ # boto3
172
+ # botocore
173
+joblib==1.4.2
174
+ # via
175
+ # librosa
176
+ # nltk
177
+ # scikit-learn
178
+jsonlines==4.0.0
179
+ # via lm-eval
180
+jsonschema==4.23.0
181
+ # via ray
182
+jsonschema-specifications==2024.10.1
183
+ # via jsonschema
184
+kiwisolver==1.4.7
185
+ # via matplotlib
186
+lazy-loader==0.4
187
+ # via librosa
188
+libnacl==2.1.0
189
+ # via tensorizer
190
+librosa==0.10.2.post1
191
+ # via -r requirements-test.in
192
+llvmlite==0.43.0
193
+ # via numba
194
+lm-eval[api]==0.4.4
195
+ # via -r requirements-test.in
196
+lxml==5.3.0
197
+ # via sacrebleu
198
+markupsafe==3.0.2
199
+ # via jinja2
200
+matplotlib==3.9.2
201
+ # via -r requirements-test.in
202
+mbstrdecoder==1.1.3
203
+ # via
204
+ # dataproperty
205
+ # pytablewriter
206
+ # typepy
207
+more-itertools==10.5.0
208
+ # via lm-eval
209
+mpmath==1.3.0
210
+ # via sympy
211
+msgpack==1.1.0
212
+ # via
213
+ # librosa
214
+ # ray
215
+multidict==6.1.0
216
+ # via
217
+ # aiohttp
218
+ # yarl
219
+multiprocess==0.70.16
220
+ # via
221
+ # datasets
222
+ # evaluate
223
+mypy-extensions==1.0.0
224
+ # via black
225
+networkx==3.2.1
226
+ # via torch
227
+nltk==3.9.1
228
+ # via rouge-score
229
+numba==0.60.0
230
+ # via librosa
231
+numexpr==2.10.1
232
+ # via lm-eval
233
+numpy==1.26.4
234
+ # via
235
+ # -r requirements-test.in
236
+ # accelerate
237
+ # bitsandbytes
238
+ # contourpy
239
+ # cupy-cuda12x
240
+ # datasets
241
+ # evaluate
242
+ # librosa
243
+ # matplotlib
244
+ # numba
245
+ # numexpr
246
+ # opencv-python
247
+ # pandas
248
+ # peft
249
+ # rouge-score
250
+ # sacrebleu
251
+ # scikit-learn
252
+ # scipy
253
+ # soxr
254
+ # tensorizer
255
+ # torchvision
256
+ # transformers
257
+nvidia-cublas-cu12==12.4.5.8
258
+ # via
259
+ # nvidia-cudnn-cu12
260
+ # nvidia-cusolver-cu12
261
+ # torch
262
+nvidia-cuda-cupti-cu12==12.4.127
263
+ # via torch
264
+nvidia-cuda-nvrtc-cu12==12.4.127
265
+ # via torch
266
+nvidia-cuda-runtime-cu12==12.4.127
267
+ # via torch
268
+nvidia-cudnn-cu12==9.1.0.70
269
+ # via torch
270
+nvidia-cufft-cu12==11.2.1.3
271
+ # via torch
272
+nvidia-curand-cu12==10.3.5.147
273
+ # via torch
274
+nvidia-cusolver-cu12==11.6.1.9
275
+ # via torch
276
+nvidia-cusparse-cu12==12.3.1.170
277
+ # via
278
+ # nvidia-cusolver-cu12
279
+ # torch
280
+nvidia-nccl-cu12==2.21.5
281
+ # via torch
282
+nvidia-nvjitlink-cu12==12.4.127
283
+ # via
284
+ # nvidia-cusolver-cu12
285
+ # nvidia-cusparse-cu12
286
+ # torch
287
+nvidia-nvtx-cu12==12.4.127
288
+ # via torch
289
+opencv-python==4.10.0.84
290
+ # via -r requirements-test.in
291
+packaging==24.1
292
+ # via
293
+ # accelerate
294
+ # black
295
+ # datamodel-code-generator
296
+ # datasets
297
+ # evaluate
298
+ # huggingface-hub
299
+ # lazy-loader
300
+ # matplotlib
301
+ # peft
302
+ # pooch
303
+ # pytest
304
+ # pytest-rerunfailures
305
+ # ray
306
+ # transformers
307
+ # typepy
308
+pandas==2.2.3
309
+ # via
310
+ # datasets
311
+ # evaluate
312
+pathspec==0.12.1
313
+ # via black
314
+pathvalidate==3.2.1
315
+ # via pytablewriter
316
+peft==0.13.2
317
+ # via
318
+ # -r requirements-test.in
319
+ # lm-eval
320
+pillow==11.0.0
321
+ # via
322
+ # matplotlib
323
+ # sentence-transformers
324
+ # torchvision
325
+platformdirs==4.3.6
326
+ # via
327
+ # black
328
+ # pooch
329
+pluggy==1.5.0
330
+ # via pytest
331
+pooch==1.8.2
332
+ # via librosa
333
+portalocker==2.10.1
334
+ # via sacrebleu
335
+propcache==0.2.0
336
+ # via yarl
337
+protobuf==5.28.3
338
+ # via
339
+ # ray
340
+ # tensorizer
341
+psutil==6.1.0
342
+ # via
343
+ # accelerate
344
+ # peft
345
+ # tensorizer
346
+py==1.11.0
347
+ # via pytest-forked
348
+pyarrow==18.0.0
349
+ # via datasets
350
+pyasn1==0.6.1
351
+ # via rsa
352
+pybind11==2.13.6
353
+ # via lm-eval
354
+pycparser==2.22
355
+ # via cffi
356
+pydantic[email]==2.9.2
357
+ # via datamodel-code-generator
358
+pydantic-core==2.23.4
359
+ # via pydantic
360
+pyparsing==3.2.0
361
+ # via matplotlib
362
+pytablewriter==1.2.0
363
+ # via lm-eval
364
+pytest==8.3.3
365
+ # via
366
+ # -r requirements-test.in
367
+ # buildkite-test-collector
368
+ # pytest-asyncio
369
+ # pytest-forked
370
+ # pytest-rerunfailures
371
+ # pytest-shard
372
+pytest-asyncio==0.24.0
373
+ # via -r requirements-test.in
374
+pytest-forked==1.6.0
375
+ # via -r requirements-test.in
376
+pytest-rerunfailures==14.0
377
+ # via -r requirements-test.in
378
+pytest-shard==0.1.2
379
+ # via -r requirements-test.in
380
+python-dateutil==2.9.0.post0
381
+ # via
382
+ # botocore
383
+ # matplotlib
384
+ # pandas
385
+ # typepy
386
+pytz==2024.2
387
+ # via
388
+ # pandas
389
+ # typepy
390
+pyyaml==6.0.2
391
+ # via
392
+ # accelerate
393
+ # awscli
394
+ # datamodel-code-generator
395
+ # datasets
396
+ # huggingface-hub
397
+ # peft
398
+ # ray
399
+ # timm
400
+ # transformers
401
+ray[adag]==2.35.0
402
+ # via -r requirements-test.in
403
+redis==5.2.0
404
+ # via tensorizer
405
+referencing==0.35.1
406
+ # via
407
+ # jsonschema
408
+ # jsonschema-specifications
409
+regex==2024.9.11
410
+ # via
411
+ # nltk
412
+ # sacrebleu
413
+ # tiktoken
414
+ # transformers
415
+requests==2.32.3
416
+ # via
417
+ # -r requirements-test.in
418
+ # buildkite-test-collector
419
+ # datasets
420
+ # evaluate
421
+ # huggingface-hub
422
+ # lm-eval
423
+ # pooch
424
+ # ray
425
+ # tiktoken
426
+ # transformers
427
+rouge-score==0.1.2
428
+ # via lm-eval
429
+rpds-py==0.20.0
430
+ # via
431
+ # jsonschema
432
+ # referencing
433
+rsa==4.7.2
434
+ # via awscli
435
+s3transfer==0.10.3
436
+ # via
437
+ # awscli
438
+ # boto3
439
+sacrebleu==2.4.3
440
+ # via lm-eval
441
+safetensors==0.4.5
442
+ # via
443
+ # accelerate
444
+ # peft
445
+ # timm
446
+ # transformers
447
+scikit-learn==1.5.2
448
+ # via
449
+ # librosa
450
+ # lm-eval
451
+ # sentence-transformers
452
+scipy==1.13.1
453
+ # via
454
+ # librosa
455
+ # scikit-learn
456
+ # sentence-transformers
457
+sentence-transformers==3.2.1
458
+ # via -r requirements-test.in
459
+six==1.16.0
460
+ # via
461
+ # python-dateutil
462
+ # rouge-score
463
+sniffio==1.3.1
464
+ # via
465
+ # anyio
466
+ # httpx
467
+soundfile==0.12.1
468
+ # via
469
+ # -r requirements-test.in
470
+ # librosa
471
+soxr==0.5.0.post1
472
+ # via librosa
473
+sqlitedict==2.1.0
474
+ # via lm-eval
475
+sympy==1.13.1
476
+ # via torch
477
+tabledata==1.3.3
478
+ # via pytablewriter
479
+tabulate==0.9.0
480
+ # via sacrebleu
481
+tcolorpy==0.1.6
482
+ # via pytablewriter
483
+tenacity==9.0.0
484
+ # via lm-eval
485
+tensorizer==2.9.0
486
+ # via -r requirements-test.in
487
+threadpoolctl==3.5.0
488
+ # via scikit-learn
489
+tiktoken==0.8.0
490
+ # via lm-eval
491
+timm==1.0.11
492
+ # via -r requirements-test.in
493
+tokenizers==0.20.1
494
+ # via transformers
495
+torch==2.5.0
496
+ # via
497
+ # -r requirements-test.in
498
+ # accelerate
499
+ # bitsandbytes
500
+ # lm-eval
501
+ # peft
502
+ # sentence-transformers
503
+ # tensorizer
504
+ # timm
505
+ # torchvision
506
+torchvision==0.20.0
507
+ # via timm
508
+tqdm==4.66.6
509
+ # via
510
+ # datasets
511
+ # evaluate
512
+ # huggingface-hub
513
+ # lm-eval
514
+ # nltk
515
+ # peft
516
+ # sentence-transformers
517
+ # tqdm-multiprocess
518
+ # transformers
519
+tqdm-multiprocess==0.0.11
520
+ # via lm-eval
521
+transformers==4.45.2
522
+ # via
523
+ # lm-eval
524
+ # peft
525
+ # sentence-transformers
526
+ # transformers-stream-generator
527
+transformers-stream-generator==0.0.5
528
+ # via -r requirements-test.in
529
+triton==3.1.0
530
+ # via torch
531
+typepy[datetime]==1.3.2
532
+ # via
533
+ # dataproperty
534
+ # pytablewriter
535
+ # tabledata
536
+typing-extensions==4.12.2
537
+ # via
538
+ # huggingface-hub
539
+ # librosa
540
+ # pydantic
541
+ # pydantic-core
542
+ # torch
543
+tzdata==2024.2
544
+ # via pandas
545
+urllib3==1.26.20
546
+ # via
547
+ # botocore
548
+ # requests
549
+word2number==1.1
550
+ # via lm-eval
551
+xxhash==3.5.0
552
+ # via
553
+ # datasets
554
+ # evaluate
555
+yarl==1.17.0
556
+ # via aiohttp
557
+zstandard==0.23.0
558
+ # via lm-eval
559
+
560
+# The following packages are considered to be unsafe in a requirements file:
561
+# setuptools