@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc74b2b232
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Updating lm-format-enforcer version and adding links to decoding libraries in docs (#4222)

Noam Gat committed Apr 20, 2024 at 11:33 UTC cc74b2b232070f74d8765a5eefa49ae93ee45490
3 files changed +8 -4
requirements-common.txt
+1 -1
@@ -12,7 +12,7 @@ uvicorn[standard]
12 pydantic >= 2.0 # Required for OpenAI server.
13 prometheus_client >= 0.18.0
14 tiktoken == 0.6.0 # Required for DBRX tokenizer
15 -lm-format-enforcer == 0.9.3
15 +lm-format-enforcer == 0.9.8
16 outlines == 0.0.34 # Requires torch >= 2.1.0
17 typing_extensions
18 filelock >= 3.10.4 # filelock starts to support `mode` argument from 3.10.4
vllm/engine/arg_utils.py
+5 -1
@@ -197,7 +197,11 @@ class EngineArgs:
197 default='outlines',
198 choices=['outlines', 'lm-format-enforcer'],
199 help='Which engine will be used for guided decoding'
200 - ' (JSON schema / regex etc).')
200 + ' (JSON schema / regex etc) by default. Currently support '
201 + 'https://github.com/outlines-dev/outlines and '
202 + 'https://github.com/noamgat/lm-format-enforcer.'
203 + ' Can be overridden per request via guided_decoding_backend'
204 + ' parameter.')
205 # Parallel arguments
206 parser.add_argument('--worker-use-ray',
207 action='store_true',
vllm/model_executor/layers/quantization/fp8.py
+2 -2
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Dict, List, Optional, Tuple
2
3 import torch
4 from torch.nn import Module
@@ -114,7 +114,7 @@ class Fp8LinearMethod(LinearMethodBase):
114 return output
115
116
117 -def per_tensor_quantize(tensor: torch.Tensor) -> tuple[torch.Tensor, float]:
117 +def per_tensor_quantize(tensor: torch.Tensor) -> Tuple[torch.Tensor, float]:
118 """Quantize a tensor using per-tensor static scaling factor.
119
120 Args: