@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d74674bbd9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] Fix arg names (#5524)

Allen.Dou committed Jun 15, 2024 at 00:47 UTC d74674bbd978fad7f27a252650249bc2550f3e92
3 files changed +6 -6
benchmarks/kernels/benchmark_paged_attention.py
+1 -1
@@ -165,7 +165,7 @@ if __name__ == '__main__':
165 choices=["v1", "v2"],
166 default="v2")
167 parser.add_argument("--batch-size", type=int, default=8)
168 - parser.add_argument("--seq_len", type=int, default=4096)
168 + parser.add_argument("--seq-len", type=int, default=4096)
169 parser.add_argument("--num-query-heads", type=int, default=64)
170 parser.add_argument("--num-kv-heads", type=int, default=8)
171 parser.add_argument("--head-size",
examples/aqlm_example.py
+1 -1
@@ -17,7 +17,7 @@ def main():
17 type=int,
18 default=0,
19 help='known good models by index, [0-4]')
20 - parser.add_argument('--tensor_parallel_size',
20 + parser.add_argument('--tensor-parallel-size',
21 '-t',
22 type=int,
23 default=1,
examples/fp8/extract_scales.py
+4 -4
@@ -327,7 +327,7 @@ if __name__ == "__main__":
327 "--quantization-param-path <filename>). This is only used "
328 "if the KV cache dtype is FP8 and on ROCm (AMD GPU).")
329 parser.add_argument(
330 - "--quantized_model",
330 + "--quantized-model",
331 help="Specify the directory containing a single quantized HF model. "
332 "It is expected that the quantization format is FP8_E4M3, for use "
333 "on ROCm (AMD GPU).",
@@ -339,18 +339,18 @@ if __name__ == "__main__":
339 choices=["auto", "safetensors", "npz", "pt"],
340 default="auto")
341 parser.add_argument(
342 - "--output_dir",
342 + "--output-dir",
343 help="Optionally specify the output directory. By default the "
344 "KV cache scaling factors will be saved in the model directory, "
345 "however you can override this behavior here.",
346 default=None)
347 parser.add_argument(
348 - "--output_name",
348 + "--output-name",
349 help="Optionally specify the output filename.",
350 # TODO: Change this once additional scaling factors are enabled
351 default="kv_cache_scales.json")
352 parser.add_argument(
353 - "--tp_size",
353 + "--tp-size",
354 help="Optionally specify the tensor-parallel (TP) size that the "
355 "quantized model should correspond to. If specified, during KV "
356 "cache scaling factor extraction the observed TP size will be "