@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8e67598aa6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] fix line length for entire codebase (#3444)

Simon Mo committed Mar 16, 2024 at 00:36 UTC 8e67598aa6ea6ce37c4c8cb470412db0ea523573
9 files changed +174 -128
.github/workflows/ruff.yml
+1 -1
@@ -28,7 +28,7 @@ jobs:
28 pip install ruff==0.1.5 codespell==2.2.6 tomli==2.0.1
29 - name: Analysing the code with ruff
30 run: |
31 - ruff vllm tests
31 + ruff .
32 - name: Spelling check with codespell
33 run: |
34 codespell --toml pyproject.toml
\ No newline at end of file
benchmarks/backend_request_func.py
+5 -3
@@ -110,7 +110,7 @@ async def async_request_vllm(
110 output.ttft = ttft
111 output.latency = time.perf_counter() - st
112
113 - # When streaming, '\0' is appended to the end of the response.
113 + # When streaming, '\0' is appended to the end of response.
114 body = data.decode("utf-8").strip("\0")
115 output.generated_text = json.loads(
116 body)["text"][0][len(request_func_input.prompt):]
@@ -192,7 +192,8 @@ async def async_request_deepspeed_mii(
192 output = RequestFuncOutput()
193 output.prompt_len = request_func_input.prompt_len
194
195 - # DeepSpeed-MII doesn't support streaming as of Jan 28 2024, will use 0 as placeholder.
195 + # DeepSpeed-MII doesn't support streaming as of Jan 28 2024,
196 + # will use 0 as placeholder.
197 # https://github.com/microsoft/DeepSpeed-MII/pull/311
198 output.ttft = 0
199
@@ -344,7 +345,8 @@ async def async_request_openai_chat_completions(
345 return output
346
347
347 -# Since vllm must support Python 3.8, we can't use str.removeprefix(prefix) introduced in Python 3.9
348 +# Since vllm must support Python 3.8, we can't use str.removeprefix(prefix)
349 +# introduced in Python 3.9
350 def remove_prefix(text: str, prefix: str) -> str:
351 if text.startswith(prefix):
352 return text[len(prefix):]
benchmarks/benchmark_prefix_caching.py
+1 -1
@@ -4,7 +4,7 @@ import time
4 from vllm import LLM
5 from vllm import SamplingParams
6
7 -PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n"
7 +PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n" # noqa: E501
8
9
10 def test_prefix(llm=None, sampling_params=None, prompts=None):
benchmarks/benchmark_serving.py
+4 -2
@@ -293,7 +293,9 @@ def main(args: argparse.Namespace):
293
294 # Save to file
295 base_model_id = model_id.split("/")[-1]
296 - file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
296 + file_name = (
297 + f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
298 + )
299 with open(file_name, "w") as outfile:
300 json.dump(result_json, outfile)
301
@@ -341,7 +343,7 @@ if __name__ == "__main__":
343 "--tokenizer",
344 type=str,
345 help=
344 - "Name or path of the tokenizer, if not using the default model tokenizer.",
346 + "Name or path of the tokenizer, if not using the default tokenizer.",
347 )
348 parser.add_argument(
349 "--best-of",
collect_env.py
+106 -75
@@ -1,3 +1,4 @@
1 +# ruff: noqa
2 # code borrowed from https://github.com/pytorch/pytorch/blob/main/torch/utils/collect_env.py
3
4 # Unlike the rest of the PyTorch this file must be python2 compliant.
@@ -11,7 +12,6 @@ import sys
12 import os
13 from collections import namedtuple
14
14 -
15 try:
16 import torch
17 TORCH_AVAILABLE = True
@@ -19,38 +19,40 @@ except (ImportError, NameError, AttributeError, OSError):
19 TORCH_AVAILABLE = False
20
21 # System Environment Information
22 -SystemEnv = namedtuple('SystemEnv', [
23 - 'torch_version',
24 - 'is_debug_build',
25 - 'cuda_compiled_version',
26 - 'gcc_version',
27 - 'clang_version',
28 - 'cmake_version',
29 - 'os',
30 - 'libc_version',
31 - 'python_version',
32 - 'python_platform',
33 - 'is_cuda_available',
34 - 'cuda_runtime_version',
35 - 'cuda_module_loading',
36 - 'nvidia_driver_version',
37 - 'nvidia_gpu_models',
38 - 'cudnn_version',
39 - 'pip_version', # 'pip' or 'pip3'
40 - 'pip_packages',
41 - 'conda_packages',
42 - 'hip_compiled_version',
43 - 'hip_runtime_version',
44 - 'miopen_runtime_version',
45 - 'caching_allocator_config',
46 - 'is_xnnpack_available',
47 - 'cpu_info',
48 - 'rocm_version', # vllm specific field
49 - 'neuron_sdk_version', # vllm specific field
50 - 'vllm_version', # vllm specific field
51 - 'vllm_build_flags', # vllm specific field
52 - 'gpu_topo', # vllm specific field
53 -])
22 +SystemEnv = namedtuple(
23 + 'SystemEnv',
24 + [
25 + 'torch_version',
26 + 'is_debug_build',
27 + 'cuda_compiled_version',
28 + 'gcc_version',
29 + 'clang_version',
30 + 'cmake_version',
31 + 'os',
32 + 'libc_version',
33 + 'python_version',
34 + 'python_platform',
35 + 'is_cuda_available',
36 + 'cuda_runtime_version',
37 + 'cuda_module_loading',
38 + 'nvidia_driver_version',
39 + 'nvidia_gpu_models',
40 + 'cudnn_version',
41 + 'pip_version', # 'pip' or 'pip3'
42 + 'pip_packages',
43 + 'conda_packages',
44 + 'hip_compiled_version',
45 + 'hip_runtime_version',
46 + 'miopen_runtime_version',
47 + 'caching_allocator_config',
48 + 'is_xnnpack_available',
49 + 'cpu_info',
50 + 'rocm_version', # vllm specific field
51 + 'neuron_sdk_version', # vllm specific field
52 + 'vllm_version', # vllm specific field
53 + 'vllm_build_flags', # vllm specific field
54 + 'gpu_topo', # vllm specific field
55 + ])
56
57 DEFAULT_CONDA_PATTERNS = {
58 "torch",
@@ -77,8 +79,10 @@ DEFAULT_PIP_PATTERNS = {
79 def run(command):
80 """Return (return-code, stdout, stderr)."""
81 shell = True if type(command) is str else False
80 - p = subprocess.Popen(command, stdout=subprocess.PIPE,
81 - stderr=subprocess.PIPE, shell=shell)
82 + p = subprocess.Popen(command,
83 + stdout=subprocess.PIPE,
84 + stderr=subprocess.PIPE,
85 + shell=shell)
86 raw_output, raw_err = p.communicate()
87 rc = p.returncode
88 if get_platform() == 'win32':
@@ -108,6 +112,7 @@ def run_and_parse_first_match(run_lambda, command, regex):
112 return None
113 return match.group(1)
114
115 +
116 def run_and_return_first_line(run_lambda, command):
117 """Run command using run_lambda and returns first line if output is not empty."""
118 rc, out, _ = run_lambda(command)
@@ -124,22 +129,23 @@ def get_conda_packages(run_lambda, patterns=None):
129 if out is None:
130 return out
131
127 - return "\n".join(
128 - line
129 - for line in out.splitlines()
130 - if not line.startswith("#")
131 - and any(name in line for name in patterns)
132 - )
132 + return "\n".join(line for line in out.splitlines()
133 + if not line.startswith("#") and any(name in line
134 + for name in patterns))
135 +
136
137 def get_gcc_version(run_lambda):
138 return run_and_parse_first_match(run_lambda, 'gcc --version', r'gcc (.*)')
139
140 +
141 def get_clang_version(run_lambda):
138 - return run_and_parse_first_match(run_lambda, 'clang --version', r'clang version (.*)')
142 + return run_and_parse_first_match(run_lambda, 'clang --version',
143 + r'clang version (.*)')
144
145
146 def get_cmake_version(run_lambda):
142 - return run_and_parse_first_match(run_lambda, 'cmake --version', r'cmake (.*)')
147 + return run_and_parse_first_match(run_lambda, 'cmake --version',
148 + r'cmake (.*)')
149
150
151 def get_nvidia_driver_version(run_lambda):
@@ -148,11 +154,13 @@ def get_nvidia_driver_version(run_lambda):
154 return run_and_parse_first_match(run_lambda, cmd,
155 r'com[.]nvidia[.]CUDA [(](.*?)[)]')
156 smi = get_nvidia_smi()
151 - return run_and_parse_first_match(run_lambda, smi, r'Driver Version: (.*?) ')
157 + return run_and_parse_first_match(run_lambda, smi,
158 + r'Driver Version: (.*?) ')
159
160
161 def get_gpu_info(run_lambda):
155 - if get_platform() == 'darwin' or (TORCH_AVAILABLE and hasattr(torch.version, 'hip') and torch.version.hip is not None):
162 + if get_platform() == 'darwin' or (TORCH_AVAILABLE and hasattr(
163 + torch.version, 'hip') and torch.version.hip is not None):
164 if TORCH_AVAILABLE and torch.cuda.is_available():
165 if torch.version.hip is not None:
166 prop = torch.cuda.get_device_properties(0)
@@ -174,7 +182,8 @@ def get_gpu_info(run_lambda):
182
183
184 def get_running_cuda_version(run_lambda):
177 - return run_and_parse_first_match(run_lambda, 'nvcc --version', r'release .+ V(.*)')
185 + return run_and_parse_first_match(run_lambda, 'nvcc --version',
186 + r'release .+ V(.*)')
187
188
189 def get_cudnn_version(run_lambda):
@@ -219,8 +228,10 @@ def get_nvidia_smi():
228 smi = 'nvidia-smi'
229 if get_platform() == 'win32':
230 system_root = os.environ.get('SYSTEMROOT', 'C:\\Windows')
222 - program_files_root = os.environ.get('PROGRAMFILES', 'C:\\Program Files')
223 - legacy_path = os.path.join(program_files_root, 'NVIDIA Corporation', 'NVSMI', smi)
231 + program_files_root = os.environ.get('PROGRAMFILES',
232 + 'C:\\Program Files')
233 + legacy_path = os.path.join(program_files_root, 'NVIDIA Corporation',
234 + 'NVSMI', smi)
235 new_path = os.path.join(system_root, 'System32', smi)
236 smis = [new_path, legacy_path]
237 for candidate_smi in smis:
@@ -232,7 +243,8 @@ def get_nvidia_smi():
243
244 def get_rocm_version(run_lambda):
245 """Returns the ROCm version if available, otherwise 'N/A'."""
235 - return run_and_parse_first_match(run_lambda, 'hipcc --version', r'HIP version: (\S+)')
246 + return run_and_parse_first_match(run_lambda, 'hipcc --version',
247 + r'HIP version: (\S+)')
248
249
250 def get_neuron_sdk_version(run_lambda):
@@ -342,13 +354,16 @@ def get_gpu_topo(run_lambda):
354 # ProcessorType=3
355 # Revision=27142
356
357 +
358 def get_cpu_info(run_lambda):
359 rc, out, err = 0, '', ''
360 if get_platform() == 'linux':
361 rc, out, err = run_lambda('lscpu')
362 elif get_platform() == 'win32':
350 - rc, out, err = run_lambda('wmic cpu get Name,Manufacturer,Family,Architecture,ProcessorType,DeviceID, \
351 - CurrentClockSpeed,MaxClockSpeed,L2CacheSize,L2CacheSpeed,Revision /VALUE')
363 + rc, out, err = run_lambda(
364 + 'wmic cpu get Name,Manufacturer,Family,Architecture,ProcessorType,DeviceID, \
365 + CurrentClockSpeed,MaxClockSpeed,L2CacheSize,L2CacheSpeed,Revision /VALUE'
366 + )
367 elif get_platform() == 'darwin':
368 rc, out, err = run_lambda("sysctl -n machdep.cpu.brand_string")
369 cpu_info = 'None'
@@ -373,18 +388,22 @@ def get_platform():
388
389
390 def get_mac_version(run_lambda):
376 - return run_and_parse_first_match(run_lambda, 'sw_vers -productVersion', r'(.*)')
391 + return run_and_parse_first_match(run_lambda, 'sw_vers -productVersion',
392 + r'(.*)')
393
394
395 def get_windows_version(run_lambda):
396 system_root = os.environ.get('SYSTEMROOT', 'C:\\Windows')
397 wmic_cmd = os.path.join(system_root, 'System32', 'Wbem', 'wmic')
398 findstr_cmd = os.path.join(system_root, 'System32', 'findstr')
383 - return run_and_read_all(run_lambda, '{} os get Caption | {} /v Caption'.format(wmic_cmd, findstr_cmd))
399 + return run_and_read_all(
400 + run_lambda,
401 + '{} os get Caption | {} /v Caption'.format(wmic_cmd, findstr_cmd))
402
403
404 def get_lsb_version(run_lambda):
387 - return run_and_parse_first_match(run_lambda, 'lsb_release -a', r'Description:\t(.*)')
405 + return run_and_parse_first_match(run_lambda, 'lsb_release -a',
406 + r'Description:\t(.*)')
407
408
409 def check_release_file(run_lambda):
@@ -443,11 +462,8 @@ def get_pip_packages(run_lambda, patterns=None):
462 # But here it is invoked as `python -mpip`
463 def run_with_pip(pip):
464 out = run_and_read_all(run_lambda, pip + ["list", "--format=freeze"])
446 - return "\n".join(
447 - line
448 - for line in out.splitlines()
449 - if any(name in line for name in patterns)
450 - )
465 + return "\n".join(line for line in out.splitlines()
466 + if any(name in line for name in patterns))
467
468 pip_version = 'pip3' if sys.version[0] == '3' else 'pip'
469 out = run_with_pip([sys.executable, '-mpip'])
@@ -472,10 +488,12 @@ def get_cuda_module_loading_config():
488 def is_xnnpack_available():
489 if TORCH_AVAILABLE:
490 import torch.backends.xnnpack
475 - return str(torch.backends.xnnpack.enabled) # type: ignore[attr-defined]
491 + return str(
492 + torch.backends.xnnpack.enabled) # type: ignore[attr-defined]
493 else:
494 return "N/A"
495
496 +
497 def get_env_info():
498 run_lambda = run
499 pip_version, pip_list_output = get_pip_packages(run_lambda)
@@ -485,9 +503,11 @@ def get_env_info():
503 debug_mode_str = str(torch.version.debug)
504 cuda_available_str = str(torch.cuda.is_available())
505 cuda_version_str = torch.version.cuda
488 - if not hasattr(torch.version, 'hip') or torch.version.hip is None: # cuda version
506 + if not hasattr(torch.version,
507 + 'hip') or torch.version.hip is None: # cuda version
508 hip_compiled_version = hip_runtime_version = miopen_runtime_version = 'N/A'
509 else: # HIP version
510 +
511 def get_version_or_na(cfg, prefix):
512 _lst = [s.rsplit(None, 1)[-1] for s in cfg if prefix in s]
513 return _lst[0] if _lst else 'N/A'
@@ -514,7 +534,9 @@ def get_env_info():
534 return SystemEnv(
535 torch_version=version_str,
536 is_debug_build=debug_mode_str,
517 - python_version='{} ({}-bit runtime)'.format(sys_version, sys.maxsize.bit_length() + 1),
537 + python_version='{} ({}-bit runtime)'.format(
538 + sys_version,
539 + sys.maxsize.bit_length() + 1),
540 python_platform=get_python_platform(),
541 is_cuda_available=cuda_available_str,
542 cuda_compiled_version=cuda_version_str,
@@ -544,6 +566,7 @@ def get_env_info():
566 gpu_topo=gpu_topo,
567 )
568
569 +
570 env_info_fmt = """
571 PyTorch version: {torch_version}
572 Is debug build: {is_debug_build}
@@ -588,6 +611,7 @@ GPU Topology:
611
612
613 def pretty_str(envinfo):
614 +
615 def replace_nones(dct, replacement='Could not collect'):
616 for key in dct.keys():
617 if dct[key] is not None:
@@ -632,9 +656,10 @@ def pretty_str(envinfo):
656 'nvidia_driver_version',
657 ]
658 all_cuda_fields = dynamic_cuda_fields + ['cudnn_version']
635 - all_dynamic_cuda_fields_missing = all(
636 - mutable_dict[field] is None for field in dynamic_cuda_fields)
637 - if TORCH_AVAILABLE and not torch.cuda.is_available() and all_dynamic_cuda_fields_missing:
659 + all_dynamic_cuda_fields_missing = all(mutable_dict[field] is None
660 + for field in dynamic_cuda_fields)
661 + if TORCH_AVAILABLE and not torch.cuda.is_available(
662 + ) and all_dynamic_cuda_fields_missing:
663 for field in all_cuda_fields:
664 mutable_dict[field] = 'No CUDA'
665 if envinfo.cuda_compiled_version is None:
@@ -647,17 +672,19 @@ def pretty_str(envinfo):
672 mutable_dict = replace_nones(mutable_dict)
673
674 # If either of these are '', replace with 'No relevant packages'
650 - mutable_dict['pip_packages'] = replace_if_empty(mutable_dict['pip_packages'])
651 - mutable_dict['conda_packages'] = replace_if_empty(mutable_dict['conda_packages'])
675 + mutable_dict['pip_packages'] = replace_if_empty(
676 + mutable_dict['pip_packages'])
677 + mutable_dict['conda_packages'] = replace_if_empty(
678 + mutable_dict['conda_packages'])
679
680 # Tag conda and pip packages with a prefix
681 # If they were previously None, they'll show up as ie '[conda] Could not collect'
682 if mutable_dict['pip_packages']:
656 - mutable_dict['pip_packages'] = prepend(mutable_dict['pip_packages'],
657 - '[{}] '.format(envinfo.pip_version))
683 + mutable_dict['pip_packages'] = prepend(
684 + mutable_dict['pip_packages'], '[{}] '.format(envinfo.pip_version))
685 if mutable_dict['conda_packages']:
659 - mutable_dict['conda_packages'] = prepend(mutable_dict['conda_packages'],
660 - '[conda] ')
686 + mutable_dict['conda_packages'] = prepend(
687 + mutable_dict['conda_packages'], '[conda] ')
688 mutable_dict['cpu_info'] = envinfo.cpu_info
689 return env_info_fmt.format(**mutable_dict)
690
@@ -671,18 +698,22 @@ def main():
698 output = get_pretty_env_info()
699 print(output)
700
674 - if TORCH_AVAILABLE and hasattr(torch, 'utils') and hasattr(torch.utils, '_crash_handler'):
701 + if TORCH_AVAILABLE and hasattr(torch, 'utils') and hasattr(
702 + torch.utils, '_crash_handler'):
703 minidump_dir = torch.utils._crash_handler.DEFAULT_MINIDUMP_DIR
704 if sys.platform == "linux" and os.path.exists(minidump_dir):
677 - dumps = [os.path.join(minidump_dir, dump) for dump in os.listdir(minidump_dir)]
705 + dumps = [
706 + os.path.join(minidump_dir, dump)
707 + for dump in os.listdir(minidump_dir)
708 + ]
709 latest = max(dumps, key=os.path.getctime)
710 ctime = os.path.getctime(latest)
680 - creation_time = datetime.datetime.fromtimestamp(ctime).strftime('%Y-%m-%d %H:%M:%S')
711 + creation_time = datetime.datetime.fromtimestamp(ctime).strftime(
712 + '%Y-%m-%d %H:%M:%S')
713 msg = "\n*** Detected a minidump at {} created on {}, ".format(latest, creation_time) + \
714 "if this is related to your bug please include it when you file a report ***"
715 print(msg, file=sys.stderr)
716
717
686 -
718 if __name__ == '__main__':
719 main()
csrc/punica/bgmv/generator.py
+1 -1
@@ -10,7 +10,7 @@ TEMPLATE = """
10 #include "bgmv_impl.cuh"
11
12 FOR_BGMV_WIDE_NARROW(INST_BGMV_TWOSIDE, {input_dtype}, {output_dtype}, {weight_dtype})
13 -""".lstrip()
13 +""".lstrip() # noqa: E501
14
15 for input_dtype in DTYPES:
16 for output_dtype in DTYPES:
examples/multilora_inference.py
+37 -32
@@ -1,5 +1,6 @@
1 """
2 -This example shows how to use the multi-LoRA functionality for offline inference.
2 +This example shows how to use the multi-LoRA functionality
3 +for offline inference.
4
5 Requires HuggingFace credentials for access to Llama2.
6 """
@@ -16,7 +17,7 @@ def create_test_prompts(
17 lora_path: str
18 ) -> List[Tuple[str, SamplingParams, Optional[LoRARequest]]]:
19 """Create a list of test prompts with their sampling parameters.
19 -
20 +
21 2 requests for base model, 4 requests for the LoRA. We define 2
22 different LoRA adapters (using the same model for demo purposes).
23 Since we also set `max_loras=1`, the expectation is that the requests
@@ -34,36 +35,40 @@ def create_test_prompts(
35 top_k=5,
36 presence_penalty=0.2,
37 max_tokens=128), None),
37 - ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
38 - SamplingParams(temperature=0.0,
39 - logprobs=1,
40 - prompt_logprobs=1,
41 - max_tokens=128,
42 - stop_token_ids=[32003]),
43 - LoRARequest("sql-lora", 1, lora_path)),
44 - ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
45 - SamplingParams(n=3,
46 - best_of=3,
47 - use_beam_search=True,
48 - temperature=0,
49 - max_tokens=128,
50 - stop_token_ids=[32003]),
51 - LoRARequest("sql-lora", 1, lora_path)),
52 - ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]",
53 - SamplingParams(temperature=0.0,
54 - logprobs=1,
55 - prompt_logprobs=1,
56 - max_tokens=128,
57 - stop_token_ids=[32003]),
58 - LoRARequest("sql-lora2", 2, lora_path)),
59 - ("[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]",
60 - SamplingParams(n=3,
61 - best_of=3,
62 - use_beam_search=True,
63 - temperature=0,
64 - max_tokens=128,
65 - stop_token_ids=[32003]),
66 - LoRARequest("sql-lora", 1, lora_path)),
38 + (
39 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
40 + SamplingParams(temperature=0.0,
41 + logprobs=1,
42 + prompt_logprobs=1,
43 + max_tokens=128,
44 + stop_token_ids=[32003]),
45 + LoRARequest("sql-lora", 1, lora_path)),
46 + (
47 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
48 + SamplingParams(n=3,
49 + best_of=3,
50 + use_beam_search=True,
51 + temperature=0,
52 + max_tokens=128,
53 + stop_token_ids=[32003]),
54 + LoRARequest("sql-lora", 1, lora_path)),
55 + (
56 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
57 + SamplingParams(temperature=0.0,
58 + logprobs=1,
59 + prompt_logprobs=1,
60 + max_tokens=128,
61 + stop_token_ids=[32003]),
62 + LoRARequest("sql-lora2", 2, lora_path)),
63 + (
64 + "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? [/user] [assistant]", # noqa: E501
65 + SamplingParams(n=3,
66 + best_of=3,
67 + use_beam_search=True,
68 + temperature=0,
69 + max_tokens=128,
70 + stop_token_ids=[32003]),
71 + LoRARequest("sql-lora", 1, lora_path)),
72 ]
73
74
examples/offline_inference_with_prefix.py
+4 -3
@@ -37,9 +37,10 @@ for output in outputs:
37
38 print("-" * 80)
39
40 -# The llm.generate call will batch all prompts and send the batch at once if resources allow.
41 -# The prefix will only be cached after the first batch is processed, so we need to call generate once
42 -# to calculate the prefix and cache it.
40 +# The llm.generate call will batch all prompts and send the batch at once
41 +# if resources allow. The prefix will only be cached after the first batch
42 +# is processed, so we need to call generate once to calculate the prefix
43 +# and cache it.
44 outputs = llm.generate(generating_prompts[0], sampling_params)
45
46 # Subsequent batches can leverage the cached prefix
setup.py
+15 -10
@@ -12,7 +12,12 @@ import setuptools
12 import sys
13 import torch
14 import torch.utils.cpp_extension as torch_cpp_ext
15 -from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
15 +from torch.utils.cpp_extension import (
16 + BuildExtension,
17 + CUDAExtension,
18 + CUDA_HOME,
19 + ROCM_HOME,
20 +)
21
22 ROOT_DIR = os.path.dirname(__file__)
23
@@ -57,9 +62,8 @@ NVCC_FLAGS = ["-O2", "-std=c++17"]
62
63 if _is_hip():
64 if ROCM_HOME is None:
60 - raise RuntimeError(
61 - "Cannot find ROCM_HOME. ROCm must be available to build the package."
62 - )
65 + raise RuntimeError("Cannot find ROCM_HOME. "
66 + "ROCm must be available to build the package.")
67 NVCC_FLAGS += ["-DUSE_ROCM"]
68 NVCC_FLAGS += ["-U__HIP_NO_HALF_CONVERSIONS__"]
69 NVCC_FLAGS += ["-U__HIP_NO_HALF_OPERATORS__"]
@@ -144,7 +148,8 @@ def get_pytorch_rocm_arch() -> Set[str]:
148 """
149 env_arch_list = os.environ.get("PYTORCH_ROCM_ARCH", None)
150
147 - # If we don't have PYTORCH_ROCM_ARCH specified pull the list from rocm_agent_enumerator
151 + # If we don't have PYTORCH_ROCM_ARCH specified pull the list from
152 + # rocm_agent_enumerator
153 if env_arch_list is None:
154 command = "rocm_agent_enumerator"
155 env_arch_list = (subprocess.check_output(
@@ -255,11 +260,11 @@ if _is_cuda():
260 "CUDA 11.1 or higher is required for compute capability 8.6.")
261 if nvcc_cuda_version < Version("11.8"):
262 if any(cc.startswith("8.9") for cc in compute_capabilities):
258 - # CUDA 11.8 is required to generate the code targeting compute capability 8.9.
259 - # However, GPUs with compute capability 8.9 can also run the code generated by
260 - # the previous versions of CUDA 11 and targeting compute capability 8.0.
261 - # Therefore, if CUDA 11.8 is not available, we target compute capability 8.0
262 - # instead of 8.9.
263 + # CUDA 11.8 is required to generate the code targeting compute
264 + # capability 8.9. However, GPUs with compute capability 8.9 can
265 + # also run the code generated by the previous versions of CUDA 11
266 + # and targeting compute capability 8.0. Therefore, if CUDA 11.8
267 + # is not available, we target compute capability 8.0 instead of 8.9.
268 warnings.warn(
269 "CUDA 11.8 or higher is required for compute capability 8.9. "
270 "Targeting compute capability 8.0 instead.",