@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6256697997

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Doc] ruff format remaining Python examples (#26795)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Oct 15, 2025 at 16:25 UTC 6256697997ee27819b66b038f71886f7fcdebf2e

21 files changed +166 -105

docs/features/quantization/auto_awq.md

+6 -4

@@ -22,13 +22,15 @@ After installing AutoAWQ, you are ready to quantize a model. Please refer to the

22 from awq import AutoAWQForCausalLM

23 from transformers import AutoTokenizer

25 - model_path = 'mistralai/Mistral-7B-Instruct-v0.2'

26 - quant_path = 'mistral-instruct-v0.2-awq'

27 - quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" }

25 + model_path = "mistralai/Mistral-7B-Instruct-v0.2"

26 + quant_path = "mistral-instruct-v0.2-awq"

27 + quant_config = {"zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM"}

29 # Load model

30 model = AutoAWQForCausalLM.from_pretrained(

31 - model_path, **{"low_cpu_mem_usage": True, "use_cache": False}

31 + model_path,

32 + low_cpu_mem_usage=True,

33 + use_cache=False,

34 )

35 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

docs/features/quantization/bitblas.md

+2 -2

@@ -34,7 +34,7 @@ llm = LLM(

34 model=model_id,

35 dtype=torch.bfloat16,

36 trust_remote_code=True,

37 - quantization="bitblas"

37 + quantization="bitblas",

38 )

39 ```

@@ -53,6 +53,6 @@ llm = LLM(

53 dtype=torch.float16,

54 trust_remote_code=True,

55 quantization="bitblas",

56 - max_model_len=1024

56 + max_model_len=1024,

57 )

58 ```

docs/features/quantization/bnb.md

+2 -2

@@ -27,7 +27,7 @@ model_id = "unsloth/tinyllama-bnb-4bit"

27 llm = LLM(

28 model=model_id,

29 dtype=torch.bfloat16,

30 - trust_remote_code=True

30 + trust_remote_code=True,

31 )

32 ```

@@ -43,7 +43,7 @@ llm = LLM(

43 model=model_id,

44 dtype=torch.bfloat16,

45 trust_remote_code=True,

46 - quantization="bitsandbytes"

46 + quantization="bitsandbytes",

47 )

48 ```

docs/features/quantization/fp8.md

+7 -2

@@ -41,7 +41,9 @@ from transformers import AutoTokenizer, AutoModelForCausalLM

42 MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"

43 model = AutoModelForCausalLM.from_pretrained(

44 - MODEL_ID, device_map="auto", torch_dtype="auto",

44 + MODEL_ID,

45 + device_map="auto",

46 + torch_dtype="auto",

47 )

48 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

49 ```

@@ -63,7 +65,10 @@ Since simple RTN does not require data for weight quantization and the activatio

66 # Configure the simple PTQ quantization

67 recipe = QuantizationModifier(

66 - targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"])

68 + targets="Linear",

69 + scheme="FP8_DYNAMIC",

70 + ignore=["lm_head"],

71 + )

73 # Apply the quantization algorithm.

74 oneshot(model=model, recipe=recipe)

docs/features/quantization/gguf.md

+7 -5

@@ -47,15 +47,15 @@ You can also use the GGUF model directly through the LLM entrypoint:

47 conversation = [

48 {

49 "role": "system",

50 - "content": "You are a helpful assistant"

50 + "content": "You are a helpful assistant",

51 },

52 {

53 "role": "user",

54 - "content": "Hello"

54 + "content": "Hello",

55 },

56 {

57 "role": "assistant",

58 - "content": "Hello! How can I assist you today?"

58 + "content": "Hello! How can I assist you today?",

59 },

60 {

61 "role": "user",

@@ -67,8 +67,10 @@ You can also use the GGUF model directly through the LLM entrypoint:

67 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)

69 # Create an LLM.

70 - llm = LLM(model="./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",

71 - tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0")

70 + llm = LLM(

71 + model="./tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",

72 + tokenizer="TinyLlama/TinyLlama-1.1B-Chat-v1.0",

73 + )

74 # Generate texts from the prompts. The output is a list of RequestOutput objects

75 # that contain the prompt, generated text, and other information.

76 outputs = llm.chat(conversation, sampling_params)

docs/features/quantization/gptqmodel.md

+1 -1

@@ -40,7 +40,7 @@ Here is an example of how to quantize `meta-llama/Llama-3.2-1B-Instruct`:

40 calibration_dataset = load_dataset(

41 "allenai/c4",

42 data_files="en/c4-train.00001-of-01024.json.gz",

43 - split="train"

43 + split="train",

44 ).select(range(1024))["text"]

46 quant_config = QuantizeConfig(bits=4, group_size=128)

docs/features/quantization/int4.md

+4 -2

@@ -39,7 +39,9 @@ from transformers import AutoTokenizer, AutoModelForCausalLM

40 MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"

41 model = AutoModelForCausalLM.from_pretrained(

42 - MODEL_ID, device_map="auto", torch_dtype="auto",

42 + MODEL_ID,

43 + device_map="auto",

44 + torch_dtype="auto",

45 )

46 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

47 ```

@@ -166,7 +168,7 @@ The following is an example of an expanded quantization recipe you can tune to y

168 },

169 ignore=["lm_head"],

170 update_size=NUM_CALIBRATION_SAMPLES,

169 - dampening_frac=0.01

171 + dampening_frac=0.01,

172 )

173 ```

174

docs/features/quantization/int8.md

+3 -1

@@ -44,7 +44,9 @@ from transformers import AutoTokenizer, AutoModelForCausalLM

45 MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"

46 model = AutoModelForCausalLM.from_pretrained(

47 - MODEL_ID, device_map="auto", torch_dtype="auto",

47 + MODEL_ID,

48 + device_map="auto",

49 + torch_dtype="auto",

50 )

51 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

52 ```

docs/features/quantization/modelopt.md

+2 -2

@@ -56,9 +56,9 @@ The quantized checkpoint can then be deployed with vLLM. As an example, the foll

56 from vllm import LLM, SamplingParams

58 def main():

59 -

59 model_id = "nvidia/Llama-3.1-8B-Instruct-FP8"

61 - # Ensure you specify quantization='modelopt' when loading the modelopt checkpoint

60 +

61 + # Ensure you specify quantization="modelopt" when loading the modelopt checkpoint

62 llm = LLM(model=model_id, quantization="modelopt", trust_remote_code=True)

64 sampling_params = SamplingParams(temperature=0.8, top_p=0.9)

docs/features/quantization/quantized_kvcache.md

+5 -3

@@ -41,9 +41,11 @@ Here is an example of how to enable FP8 quantization:

41 from vllm import LLM, SamplingParams

43 sampling_params = SamplingParams(temperature=0.7, top_p=0.8)

44 - llm = LLM(model="meta-llama/Llama-2-7b-chat-hf",

45 - kv_cache_dtype="fp8",

46 - calculate_kv_scales=True)

44 + llm = LLM(

45 + model="meta-llama/Llama-2-7b-chat-hf",

46 + kv_cache_dtype="fp8",

47 + calculate_kv_scales=True,

48 + )

49 prompt = "London is the capital of"

50 out = llm.generate(prompt, sampling_params)[0].outputs[0].text

51 print(out)

docs/features/quantization/quark.md

+44 -21

@@ -48,7 +48,9 @@ to fetch model and tokenizer.

48 MAX_SEQ_LEN = 512

50 model = AutoModelForCausalLM.from_pretrained(

51 - MODEL_ID, device_map="auto", torch_dtype="auto",

51 + MODEL_ID,

52 + device_map="auto",

53 + torch_dtype="auto",

54 )

55 model.eval()

@@ -75,10 +77,18 @@ to [Adding Calibration Datasets](https://quark.docs.amd.com/latest/pytorch/calib

77 dataset = load_dataset("mit-han-lab/pile-val-backup", split="validation")

78 text_data = dataset["text"][:NUM_CALIBRATION_DATA]

78 - tokenized_outputs = tokenizer(text_data, return_tensors="pt",

79 - padding=True, truncation=True, max_length=MAX_SEQ_LEN)

80 - calib_dataloader = DataLoader(tokenized_outputs['input_ids'],

81 - batch_size=BATCH_SIZE, drop_last=True)

80 + tokenized_outputs = tokenizer(

81 + text_data,

82 + return_tensors="pt",

83 + padding=True,

84 + truncation=True,

85 + max_length=MAX_SEQ_LEN,

86 + )

87 + calib_dataloader = DataLoader(

88 + tokenized_outputs['input_ids'],

89 + batch_size=BATCH_SIZE,

90 + drop_last=True,

91 + )

92 ```

94 ### 3. Set the Quantization Configuration

@@ -103,26 +113,32 @@ kv-cache and the quantization algorithm is AutoSmoothQuant.

113 load_quant_algo_config_from_file)

114

115 # Define fp8/per-tensor/static spec.

106 - FP8_PER_TENSOR_SPEC = FP8E4M3PerTensorSpec(observer_method="min_max",

107 - is_dynamic=False).to_quantization_spec()

116 + FP8_PER_TENSOR_SPEC = FP8E4M3PerTensorSpec(

117 + observer_method="min_max",

118 + is_dynamic=False,

119 + ).to_quantization_spec()

120

121 # Define global quantization config, input tensors and weight apply FP8_PER_TENSOR_SPEC.

110 - global_quant_config = QuantizationConfig(input_tensors=FP8_PER_TENSOR_SPEC,

111 - weight=FP8_PER_TENSOR_SPEC)

122 + global_quant_config = QuantizationConfig(

123 + input_tensors=FP8_PER_TENSOR_SPEC,

124 + weight=FP8_PER_TENSOR_SPEC,

125 + )

126

127 # Define quantization config for kv-cache layers, output tensors apply FP8_PER_TENSOR_SPEC.

128 KV_CACHE_SPEC = FP8_PER_TENSOR_SPEC

129 kv_cache_layer_names_for_llama = ["*k_proj", "*v_proj"]

116 - kv_cache_quant_config = {name :

117 - QuantizationConfig(input_tensors=global_quant_config.input_tensors,

118 - weight=global_quant_config.weight,

119 - output_tensors=KV_CACHE_SPEC)

120 - for name in kv_cache_layer_names_for_llama}

130 + kv_cache_quant_config = {

131 + name: QuantizationConfig(

132 + input_tensors=global_quant_config.input_tensors,

133 + weight=global_quant_config.weight,

134 + output_tensors=KV_CACHE_SPEC,

135 + )

136 + for name in kv_cache_layer_names_for_llama

137 + }

138 layer_quant_config = kv_cache_quant_config.copy()

139

140 # Define algorithm config by config file.

124 - LLAMA_AUTOSMOOTHQUANT_CONFIG_FILE =

125 - 'examples/torch/language_modeling/llm_ptq/models/llama/autosmoothquant_config.json'

141 + LLAMA_AUTOSMOOTHQUANT_CONFIG_FILE = "examples/torch/language_modeling/llm_ptq/models/llama/autosmoothquant_config.json"

142 algo_config = load_quant_algo_config_from_file(LLAMA_AUTOSMOOTHQUANT_CONFIG_FILE)

143

144 EXCLUDE_LAYERS = ["lm_head"]

@@ -131,7 +147,8 @@ kv-cache and the quantization algorithm is AutoSmoothQuant.

147 layer_quant_config=layer_quant_config,

148 kv_cache_quant_config=kv_cache_quant_config,

149 exclude=EXCLUDE_LAYERS,

134 - algo_config=algo_config)

150 + algo_config=algo_config,

151 + )

152 ```

153

154 ### 4. Quantize the Model and Export

@@ -165,8 +182,11 @@ for more exporting format details.

182 EXPORT_DIR = MODEL_ID.split("/")[1] + "-w-fp8-a-fp8-kvcache-fp8-pertensor-autosmoothquant"

183 exporter = ModelExporter(config=export_config, export_dir=EXPORT_DIR)

184 with torch.no_grad():

168 - exporter.export_safetensors_model(freezed_model,

169 - quant_config=quant_config, tokenizer=tokenizer)

185 + exporter.export_safetensors_model(

186 + freezed_model,

187 + quant_config=quant_config,

188 + tokenizer=tokenizer,

189 + )

190 ```

191

192 ### 5. Evaluation in vLLM

@@ -189,8 +209,11 @@ Now, you can load and run the Quark quantized model directly through the LLM ent

209 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)

210

211 # Create an LLM.

192 - llm = LLM(model="Llama-2-70b-chat-hf-w-fp8-a-fp8-kvcache-fp8-pertensor-autosmoothquant",

193 - kv_cache_dtype='fp8',quantization='quark')

212 + llm = LLM(

213 + model="Llama-2-70b-chat-hf-w-fp8-a-fp8-kvcache-fp8-pertensor-autosmoothquant",

214 + kv_cache_dtype="fp8",

215 + quantization="quark",

216 + )

217 # Generate texts from the prompts. The output is a list of RequestOutput objects

218 # that contain the prompt, generated text, and other information.

219 outputs = llm.generate(prompts, sampling_params)

docs/getting_started/quickstart.md

+5 -3

@@ -194,8 +194,10 @@ Since this server is compatible with OpenAI API, you can use it as a drop-in rep

194 api_key=openai_api_key,

195 base_url=openai_api_base,

196 )

197 - completion = client.completions.create(model="Qwen/Qwen2.5-1.5B-Instruct",

198 - prompt="San Francisco is a")

197 + completion = client.completions.create(

198 + model="Qwen/Qwen2.5-1.5B-Instruct",

199 + prompt="San Francisco is a",

200 + )

201 print("Completion result:", completion)

202 ```

203

@@ -239,7 +241,7 @@ Alternatively, you can use the `openai` Python package:

241 messages=[

242 {"role": "system", "content": "You are a helpful assistant."},

243 {"role": "user", "content": "Tell me a joke."},

242 - ]

244 + ],

245 )

246 print("Chat response:", chat_response)

247 ```

docs/models/extensions/tensorizer.md

+2 -2

@@ -60,7 +60,7 @@ from vllm import LLM

60 llm = LLM(

61 "s3://my-bucket/vllm/facebook/opt-125m/v1",

62 load_format="tensorizer",

63 - enable_lora=True

63 + enable_lora=True,

64 )

65 ```

@@ -97,6 +97,6 @@ llm = LLM(

97 "s3://my-bucket/vllm/facebook/opt-125m/v1",

98 load_format="tensorizer",

99 enable_lora=True,

100 - model_loader_extra_config={"deserialization_kwargs": {"num_readers": 2}}

100 + model_loader_extra_config={"deserialization_kwargs": {"num_readers": 2}},

101 )

102 ```

docs/models/generative_models.md

+3 -3

@@ -98,15 +98,15 @@ and automatically applies the model's [chat template](https://huggingface.co/doc

98 conversation = [

99 {

100 "role": "system",

101 - "content": "You are a helpful assistant"

101 + "content": "You are a helpful assistant",

102 },

103 {

104 "role": "user",

105 - "content": "Hello"

105 + "content": "Hello",

106 },

107 {

108 "role": "assistant",

109 - "content": "Hello! How can I assist you today?"

109 + "content": "Hello! How can I assist you today?",

110 },

111 {

112 "role": "user",

docs/models/pooling_models.md

+16 -10

@@ -130,8 +130,10 @@ It is designed for embedding models and cross-encoder models. Embedding models u

130 from vllm import LLM

131

132 llm = LLM(model="BAAI/bge-reranker-v2-m3", runner="pooling")

133 -(output,) = llm.score("What is the capital of France?",

134 - "The capital of Brazil is Brasilia.")

133 +(output,) = llm.score(

134 + "What is the capital of France?",

135 + "The capital of Brazil is Brasilia.",

136 +)

137

138 score = output.outputs.score

139 print(f"Score: {score}")

@@ -209,7 +211,7 @@ For models that support Matryoshka Embeddings but not recognized by vLLM, please

211

212 Here is an example to serve a model with Matryoshka Embeddings enabled.

213

212 -```text

214 +```bash

215 vllm serve Snowflake/snowflake-arctic-embed-m-v1.5 --hf-overrides '{"matryoshka_dimensions":[256]}'

216 ```

217

@@ -220,11 +222,15 @@ You can change the output dimensions of embedding models that support Matryoshka

222 ```python

223 from vllm import LLM, PoolingParams

224

223 -llm = LLM(model="jinaai/jina-embeddings-v3",

224 - runner="pooling",

225 - trust_remote_code=True)

226 -outputs = llm.embed(["Follow the white rabbit."],

227 - pooling_params=PoolingParams(dimensions=32))

225 +llm = LLM(

226 + model="jinaai/jina-embeddings-v3",

227 + runner="pooling",

228 + trust_remote_code=True,

229 +)

230 +outputs = llm.embed(

231 + ["Follow the white rabbit."],

232 + pooling_params=PoolingParams(dimensions=32),

233 +)

234 print(outputs[0].outputs)

235 ```

236

@@ -234,13 +240,13 @@ A code example can be found here: <gh-file:examples/offline_inference/pooling/em

240

241 Use the following command to start vllm server.

242

237 -```text

243 +```bash

244 vllm serve jinaai/jina-embeddings-v3 --trust-remote-code

245 ```

246

247 You can change the output dimensions of embedding models that support Matryoshka Embeddings by using the dimensions parameter.

248

243 -```text

249 +```bash

250 curl http://127.0.0.1:8000/v1/embeddings \

251 -H 'accept: application/json' \

252 -H 'Content-Type: application/json' \

docs/models/supported_models.md

+2 -2

@@ -278,8 +278,8 @@ https_proxy=http://your.proxy.server:port vllm serve <model_name>

278 ```python

279 import os

280

281 -os.environ['http_proxy'] = 'http://your.proxy.server:port'

282 -os.environ['https_proxy'] = 'http://your.proxy.server:port'

281 +os.environ["http_proxy"] = "http://your.proxy.server:port"

282 +os.environ["https_proxy"] = "http://your.proxy.server:port"

283 ```

284

285 ### ModelScope

docs/serving/expert_parallel_deployment.md

+3 -3

@@ -243,10 +243,10 @@ try:

243 "remote_engine_id": None, # Will be populated by vLLM

244 "remote_block_ids": None, # Will be populated by vLLM

245 "remote_host": None, # Will be populated by vLLM

246 - "remote_port": None # Will be populated by vLLM

246 + "remote_port": None, # Will be populated by vLLM

247 }

248 },

249 - extra_headers={"X-Request-Id": request_id}

249 + extra_headers={"X-Request-Id": request_id},

250 )

251

252 print("-" * 50)

@@ -262,7 +262,7 @@ try:

262 extra_body={

263 "kv_transfer_params": prefill_response.kv_transfer_params # Pass KV cache info

264 },

265 - extra_headers={"X-Request-Id": request_id} # Same request ID

265 + extra_headers={"X-Request-Id": request_id}, # Same request ID

266 )

267

268 print("-" * 50)

docs/serving/integrations/langchain.md

+9 -7

@@ -15,13 +15,15 @@ To run inference on a single or multiple GPUs, use `VLLM` class from `langchain`

15 ```python

16 from langchain_community.llms import VLLM

18 - llm = VLLM(model="mosaicml/mpt-7b",

19 - trust_remote_code=True, # mandatory for hf models

20 - max_new_tokens=128,

21 - top_k=10,

22 - top_p=0.95,

23 - temperature=0.8,

24 - # tensor_parallel_size=... # for distributed inference

18 + llm = VLLM(

19 + model="mosaicml/mpt-7b",

20 + trust_remote_code=True, # mandatory for hf models

21 + max_new_tokens=128,

22 + top_k=10,

23 + top_p=0.95,

24 + temperature=0.8,

25 + # for distributed inference

26 + # tensor_parallel_size=...,

27 )

29 print(llm("What is the capital of France ?"))

docs/serving/openai_compatible_server.md

+31 -26

@@ -24,8 +24,8 @@ To call the server, in your preferred text editor, create a script that uses an

24 completion = client.chat.completions.create(

25 model="NousResearch/Meta-Llama-3-8B-Instruct",

26 messages=[

27 - {"role": "user", "content": "Hello!"}

28 - ]

27 + {"role": "user", "content": "Hello!"},

28 + ],

29 )

31 print(completion.choices[0].message)

@@ -101,8 +101,13 @@ both a `type` and a `text` field. An example is provided below:

101 completion = client.chat.completions.create(

102 model="NousResearch/Meta-Llama-3-8B-Instruct",

103 messages=[

104 - {"role": "user", "content": [{"type": "text", "text": "Classify this sentiment: vLLM is wonderful!"}]}

105 - ]

104 + {

105 + "role": "user",

106 + "content": [

107 + {"type": "text", "text": "Classify this sentiment: vLLM is wonderful!"},

108 + ],

109 + },

110 + ],

111 )

112 ```

113

@@ -130,11 +135,11 @@ Or directly merge them into the JSON payload if you are using HTTP call directly

135 completion = client.chat.completions.create(

136 model="NousResearch/Meta-Llama-3-8B-Instruct",

137 messages=[

133 - {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}

138 + {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"},

139 ],

140 extra_body={

136 - "structured_outputs": {"choice": ["positive", "negative"]}

137 - }

141 + "structured_outputs": {"choice": ["positive", "negative"]},

142 + },

143 )

144 ```

145

@@ -149,11 +154,11 @@ with `--enable-request-id-headers`.

154 completion = client.chat.completions.create(

155 model="NousResearch/Meta-Llama-3-8B-Instruct",

156 messages=[

152 - {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"}

157 + {"role": "user", "content": "Classify this sentiment: vLLM is wonderful!"},

158 ],

159 extra_headers={

160 "x-request-id": "sentiment-classification-00001",

156 - }

161 + },

162 )

163 print(completion._request_id)

164

@@ -162,7 +167,7 @@ with `--enable-request-id-headers`.

167 prompt="A robot may not injure a human being",

168 extra_headers={

169 "x-request-id": "completion-test",

165 - }

170 + },

171 )

172 print(completion._request_id)

173 ```

@@ -403,7 +408,7 @@ The Transcriptions API supports uploading audio files in various formats includi

408 model="openai/whisper-large-v3-turbo",

409 file=audio_file,

410 language="en",

406 - response_format="verbose_json"

411 + response_format="verbose_json",

412 )

413

414 print(transcription.text)

@@ -812,22 +817,22 @@ You can pass multi-modal inputs to scoring models by passing `content` including

817 "model": "jinaai/jina-reranker-m0",

818 "text_1": "slm markdown",

819 "text_2": {

815 - "content": [

816 - {

817 - "type": "image_url",

818 - "image_url": {

819 - "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"

820 - },

821 - },

822 - {

823 - "type": "image_url",

824 - "image_url": {

825 - "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"

826 - },

827 - },

828 - ]

829 - }

820 + "content": [

821 + {

822 + "type": "image_url",

823 + "image_url": {

824 + "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"

825 + },

826 + },

827 + {

828 + "type": "image_url",

829 + "image_url": {

830 + "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"

831 + },

832 + },

833 + ],

834 },

835 + },

836 )

837 response.raise_for_status()

838 response_json = response.json()

examples/offline_inference/openai_batch/README.md

+11 -3

@@ -152,7 +152,9 @@ def generate_presigned_url(s3_client, client_method, method_parameters, expires_

152 """

153 try:

154 url = s3_client.generate_presigned_url(

155 - ClientMethod=client_method, Params=method_parameters, ExpiresIn=expires_in

155 + ClientMethod=client_method,

156 + Params=method_parameters,

157 + ExpiresIn=expires_in,

158 )

159 except ClientError:

160 raise

@@ -161,10 +163,16 @@ def generate_presigned_url(s3_client, client_method, method_parameters, expires_

163

164 s3_client = boto3.client("s3")

165 input_url = generate_presigned_url(

164 - s3_client, "get_object", {"Bucket": "MY_BUCKET", "Key": "MY_INPUT_FILE.jsonl"}, 3600

166 + s3_client,

167 + "get_object",

168 + {"Bucket": "MY_BUCKET", "Key": "MY_INPUT_FILE.jsonl"},

169 + expires_in=3600,

170 )

171 output_url = generate_presigned_url(

167 - s3_client, "put_object", {"Bucket": "MY_BUCKET", "Key": "MY_OUTPUT_FILE.jsonl"}, 3600

172 + s3_client,

173 + "put_object",

174 + {"Bucket": "MY_BUCKET", "Key": "MY_OUTPUT_FILE.jsonl"},

175 + expires_in=3600,

176 )

177 print(f"{input_url=}")

178 print(f"{output_url=}")

examples/others/tensorize_vllm_model.py

+1 -1

@@ -84,7 +84,7 @@ directly to load models:

84 from vllm import LLM

85 llm = LLM(

86 "s3://my-bucket/vllm/facebook/opt-125m/v1",

87 - load_format="tensorizer"

87 + load_format="tensorizer",

88 )

89 ```