A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Misc] Fix arg names (#5524)
Allen.Dou committed
Jun 15, 2024 at 00:47 UTC
d74674bbd978fad7f27a252650249bc2550f3e92
3 files changed
+6
-6
benchmarks/kernels/benchmark_paged_attention.py
+1
-1
@@ -165,7 +165,7 @@ if __name__ == '__main__':
165
choices=["v1", "v2"],
166
default="v2")
167
parser.add_argument("--batch-size", type=int, default=8)
168
- parser.add_argument("--seq_len", type=int, default=4096)
168
+ parser.add_argument("--seq-len", type=int, default=4096)
169
parser.add_argument("--num-query-heads", type=int, default=64)
170
parser.add_argument("--num-kv-heads", type=int, default=8)
171
parser.add_argument("--head-size",
examples/aqlm_example.py
+1
-1
@@ -17,7 +17,7 @@ def main():
17
type=int,
18
default=0,
19
help='known good models by index, [0-4]')
20
- parser.add_argument('--tensor_parallel_size',
20
+ parser.add_argument('--tensor-parallel-size',
21
'-t',
22
type=int,
23
default=1,
examples/fp8/extract_scales.py
+4
-4
@@ -327,7 +327,7 @@ if __name__ == "__main__":
327
"--quantization-param-path <filename>). This is only used "
328
"if the KV cache dtype is FP8 and on ROCm (AMD GPU).")
329
parser.add_argument(
330
- "--quantized_model",
330
+ "--quantized-model",
331
help="Specify the directory containing a single quantized HF model. "
332
"It is expected that the quantization format is FP8_E4M3, for use "
333
"on ROCm (AMD GPU).",
@@ -339,18 +339,18 @@ if __name__ == "__main__":
339
choices=["auto", "safetensors", "npz", "pt"],
340
default="auto")
341
parser.add_argument(
342
- "--output_dir",
342
+ "--output-dir",
343
help="Optionally specify the output directory. By default the "
344
"KV cache scaling factors will be saved in the model directory, "
345
"however you can override this behavior here.",
346
default=None)
347
parser.add_argument(
348
- "--output_name",
348
+ "--output-name",
349
help="Optionally specify the output filename.",
350
# TODO: Change this once additional scaling factors are enabled
351
default="kv_cache_scales.json")
352
parser.add_argument(
353
- "--tp_size",
353
+ "--tp-size",
354
help="Optionally specify the tensor-parallel (TP) size that the "
355
"quantized model should correspond to. If specified, during KV "
356
"cache scaling factor extraction the observed TP size will be "