@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55d6361b13
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Misc] Fix arg names in quantizer script (#5507)

Allen.Dou committed Jun 14, 2024 at 10:02 UTC 55d6361b13ae6328de809f57a69b719c1600040a
1 file changed +8 -8
examples/fp8/quantizer/quantize.py
+8 -8
@@ -332,7 +332,7 @@ def main(args):
332
333 if __name__ == "__main__":
334 parser = argparse.ArgumentParser(description=__doc__)
335 - parser.add_argument("--model_dir",
335 + parser.add_argument("--model-dir",
336 help="Specify where the HuggingFace model is",
337 required=True)
338 parser.add_argument("--device", default="cuda")
@@ -346,19 +346,19 @@ if __name__ == "__main__":
346 "full_prec"
347 ],
348 )
349 - parser.add_argument("--batch_size",
349 + parser.add_argument("--batch-size",
350 help="Batch size for calibration.",
351 type=int,
352 default=1)
353 - parser.add_argument("--calib_size",
353 + parser.add_argument("--calib-size",
354 help="Number of samples for calibration.",
355 type=int,
356 default=512)
357 - parser.add_argument("--output_dir", default="exported_model")
358 - parser.add_argument("--tp_size", type=int, default=1)
359 - parser.add_argument("--pp_size", type=int, default=1)
360 - parser.add_argument("--awq_block_size", type=int, default=128)
361 - parser.add_argument("--kv_cache_dtype",
357 + parser.add_argument("--output-dir", default="exported_model")
358 + parser.add_argument("--tp-size", type=int, default=1)
359 + parser.add_argument("--pp-size", type=int, default=1)
360 + parser.add_argument("--awq-block-size", type=int, default=128)
361 + parser.add_argument("--kv-cache-dtype",
362 help="KV Cache dtype.",
363 default=None,
364 choices=["int8", "fp8", None])