A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Updating lm-format-enforcer version and adding links to decoding libraries in docs (#4222)
Noam Gat committed
Apr 20, 2024 at 11:33 UTC
cc74b2b232070f74d8765a5eefa49ae93ee45490
3 files changed
+8
-4
requirements-common.txt
+1
-1
@@ -12,7 +12,7 @@ uvicorn[standard]
12
pydantic >= 2.0 # Required for OpenAI server.
13
prometheus_client >= 0.18.0
14
tiktoken == 0.6.0 # Required for DBRX tokenizer
15
-lm-format-enforcer == 0.9.3
15
+lm-format-enforcer == 0.9.8
16
outlines == 0.0.34 # Requires torch >= 2.1.0
17
typing_extensions
18
filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
vllm/engine/arg_utils.py
+5
-1
@@ -197,7 +197,11 @@ class EngineArgs:
197
default='outlines',
198
choices=['outlines', 'lm-format-enforcer'],
199
help='Which engine will be used for guided decoding'
200
- ' (JSON schema / regex etc).')
200
+ ' (JSON schema / regex etc) by default. Currently support '
201
+ 'https://github.com/outlines-dev/outlines and '
202
+ 'https://github.com/noamgat/lm-format-enforcer.'
203
+ ' Can be overridden per request via guided_decoding_backend'
204
+ ' parameter.')
205
# Parallel arguments
206
parser.add_argument('--worker-use-ray',
207
action='store_true',
vllm/model_executor/layers/quantization/fp8.py
+2
-2
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Dict, List, Optional, Tuple
2
3
import torch
4
from torch.nn import Module
@@ -114,7 +114,7 @@ class Fp8LinearMethod(LinearMethodBase):
114
return output
115
116
117
-def per_tensor_quantize(tensor: torch.Tensor) -> tuple[torch.Tensor, float]:
117
+def per_tensor_quantize(tensor: torch.Tensor) -> Tuple[torch.Tensor, float]:
118
"""Quantize a tensor using per-tensor static scaling factor.
119
120
Args: