@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 87054a57ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Doc]: Update the Json Example of the `Engine Arguments` document (#12045)

maang-h committed Jan 15, 2025 at 01:03 UTC 87054a57ab39bad6c7fe8999e7d93566ded713e3
1 file changed +4 -4
vllm/engine/arg_utils.py
+4 -4
@@ -538,7 +538,7 @@ class EngineArgs:
538 default=None,
539 type=json.loads,
540 help='RoPE scaling configuration in JSON format. '
541 - 'For example, {"rope_type":"dynamic","factor":2.0}')
541 + 'For example, ``{"rope_type":"dynamic","factor":2.0}``')
542 parser.add_argument('--rope-theta',
543 default=None,
544 type=float,
@@ -607,7 +607,7 @@ class EngineArgs:
607 default=None,
608 type=json.loads,
609 help=('Overrides for the multimodal input mapping/processing, '
610 - 'e.g., image processor. For example: {"num_crops": 4}.'))
610 + 'e.g., image processor. For example: ``{"num_crops": 4}``.'))
611 parser.add_argument(
612 '--disable-mm-preprocessor-cache',
613 action='store_true',
@@ -908,13 +908,13 @@ class EngineArgs:
908 type=json.loads,
909 default=None,
910 help="Override or set neuron device configuration. "
911 - "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
911 + "e.g. ``{\"cast_logits_dtype\": \"bloat16\"}``.")
912 parser.add_argument(
913 '--override-pooler-config',
914 type=PoolerConfig.from_json,
915 default=None,
916 help="Override or set the pooling method for pooling models. "
917 - "e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
917 + "e.g. ``{\"pooling_type\": \"mean\", \"normalize\": false}``.")
918
919 parser.add_argument('--compilation-config',
920 '-O',