A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Doc]: Update the Json Example of the `Engine Arguments` document (#12045)
maang-h committed
Jan 15, 2025 at 01:03 UTC
87054a57ab39bad6c7fe8999e7d93566ded713e3
1 file changed
+4
-4
vllm/engine/arg_utils.py
+4
-4
@@ -538,7 +538,7 @@ class EngineArgs:
538
default=None,
539
type=json.loads,
540
help='RoPE scaling configuration in JSON format. '
541
- 'For example, {"rope_type":"dynamic","factor":2.0}')
541
+ 'For example, ``{"rope_type":"dynamic","factor":2.0}``')
542
parser.add_argument('--rope-theta',
543
default=None,
544
type=float,
@@ -607,7 +607,7 @@ class EngineArgs:
607
default=None,
608
type=json.loads,
609
help=('Overrides for the multimodal input mapping/processing, '
610
- 'e.g., image processor. For example: {"num_crops": 4}.'))
610
+ 'e.g., image processor. For example: ``{"num_crops": 4}``.'))
611
parser.add_argument(
612
'--disable-mm-preprocessor-cache',
613
action='store_true',
@@ -908,13 +908,13 @@ class EngineArgs:
908
type=json.loads,
909
default=None,
910
help="Override or set neuron device configuration. "
911
- "e.g. {\"cast_logits_dtype\": \"bloat16\"}.'")
911
+ "e.g. ``{\"cast_logits_dtype\": \"bloat16\"}``.")
912
parser.add_argument(
913
'--override-pooler-config',
914
type=PoolerConfig.from_json,
915
default=None,
916
help="Override or set the pooling method for pooling models. "
917
- "e.g. {\"pooling_type\": \"mean\", \"normalize\": false}.'")
917
+ "e.g. ``{\"pooling_type\": \"mean\", \"normalize\": false}``.")
918
919
parser.add_argument('--compilation-config',
920
'-O',