@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 974dfd4971
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Model] IBM/NASA Prithvi Geospatial model (#12830)

Christian Pinto committed Feb 12, 2025 at 04:34 UTC 974dfd497149e871e59e35b677a85cca66ec3bae
7 files changed +811 -9
examples/offline_inference/prithvi_geospatial_mae.py new
+530
@@ -0,0 +1,530 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""
3 +This is a demo script showing how to use the
4 +PrithviGeospatialMAE model with vLLM
5 +This script is based on: https://huggingface.co/ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11/blob/main/inference.py # noqa
6 +
7 +Target model weights: https://huggingface.co/ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11/resolve/main/Prithvi-EO-V2-300M-TL-Sen1Floods11.pt # noqa
8 +
9 +The requirements for running this script are:
10 +- Installing [terratorch, albumentations, rasterio] in your python environment
11 +- downloading the model weights in a 'model' folder local to the script
12 + (temporary measure until the proper config.json file is uploaded to HF)
13 +- download an input example image (India_900498_S2Hand.tif) and place it in
14 + the same folder with the script (or specify with the --data_file argument)
15 +
16 +Run the example:
17 +python prithvi_geospatial_mae.py
18 +
19 +""" # noqa: E501
20 +import argparse
21 +import datetime
22 +import os
23 +import re
24 +from typing import List, Union
25 +
26 +import albumentations
27 +import numpy as np
28 +import rasterio
29 +import torch
30 +from einops import rearrange
31 +from terratorch.datamodules import Sen1Floods11NonGeoDataModule
32 +
33 +from vllm import LLM
34 +
35 +NO_DATA = -9999
36 +NO_DATA_FLOAT = 0.0001
37 +OFFSET = 0
38 +PERCENTILE = 99
39 +
40 +model_config = """{
41 + "architectures": ["PrithviGeoSpatialMAE"],
42 + "num_classes": 0,
43 + "pretrained_cfg": {
44 + "task_args": {
45 + "task": "SemanticSegmentationTask",
46 + "model_factory": "EncoderDecoderFactory",
47 + "loss": "ce",
48 + "ignore_index": -1,
49 + "lr": 0.001,
50 + "freeze_backbone": false,
51 + "freeze_decoder": false,
52 + "plot_on_val": 10,
53 + "optimizer": "AdamW",
54 + "scheduler": "CosineAnnealingLR"
55 + },
56 + "model_args": {
57 + "backbone_pretrained": false,
58 + "backbone": "prithvi_eo_v2_300_tl",
59 + "decoder": "UperNetDecoder",
60 + "decoder_channels": 256,
61 + "decoder_scale_modules": true,
62 + "num_classes": 2,
63 + "rescale": true,
64 + "backbone_bands": [
65 + "BLUE",
66 + "GREEN",
67 + "RED",
68 + "NIR_NARROW",
69 + "SWIR_1",
70 + "SWIR_2"
71 + ],
72 + "head_dropout": 0.1,
73 + "necks": [
74 + {
75 + "name": "SelectIndices",
76 + "indices": [
77 + 5,
78 + 11,
79 + 17,
80 + 23
81 + ]
82 + },
83 + {
84 + "name": "ReshapeTokensToImage"
85 + }
86 + ]
87 + },
88 + "optimizer_params" : {
89 + "lr": 5.0e-05,
90 + "betas": [0.9, 0.999],
91 + "eps": [1.0e-08],
92 + "weight_decay": 0.05,
93 + "amsgrad": false,
94 + "maximize": false,
95 + "capturable": false,
96 + "differentiable": false
97 + },
98 + "scheduler_params" : {
99 + "T_max": 50,
100 + "eta_min": 0,
101 + "last_epoch": -1,
102 + "verbose": "deprecated"
103 + }
104 + },
105 +
106 +
107 + "torch_dtype": "float32"
108 +}
109 +"""
110 +
111 +# Temporarily creating the "config.json" for the model.
112 +# This is going to disappear once the correct config.json is available on HF
113 +with open(os.path.join(os.path.dirname(__file__), "./model/config.json"),
114 + 'w') as config_file:
115 + config_file.write(model_config)
116 +
117 +datamodule_config = {
118 + 'bands': ['BLUE', 'GREEN', 'RED', 'NIR_NARROW', 'SWIR_1', 'SWIR_2'],
119 + 'batch_size':
120 + 16,
121 + 'constant_scale':
122 + 0.0001,
123 + 'data_root':
124 + '/dccstor/geofm-finetuning/datasets/sen1floods11',
125 + 'drop_last':
126 + True,
127 + 'no_data_replace':
128 + 0.0,
129 + 'no_label_replace':
130 + -1,
131 + 'num_workers':
132 + 8,
133 + 'test_transform': [
134 + albumentations.Resize(always_apply=False,
135 + height=448,
136 + interpolation=1,
137 + p=1,
138 + width=448),
139 + albumentations.pytorch.ToTensorV2(transpose_mask=False,
140 + always_apply=True,
141 + p=1.0)
142 + ],
143 +}
144 +
145 +
146 +class PrithviMAE:
147 +
148 + def __init__(self):
149 + print("Initializing PrithviMAE model")
150 + self.model = LLM(model=os.path.join(os.path.dirname(__file__),
151 + "./model"),
152 + skip_tokenizer_init=True,
153 + dtype="float32")
154 +
155 + def run(self, input_data, location_coords):
156 + print("################ Running inference on vLLM ##############")
157 + # merge the inputs into one data structure
158 + mm_data = {
159 + "pixel_values":
160 + torch.empty(0) if input_data is None else input_data,
161 + "location_coords":
162 + torch.empty(0) if location_coords is None else location_coords
163 + }
164 +
165 + prompt = {"prompt_token_ids": [1], "multi_modal_data": mm_data}
166 +
167 + outputs = self.model.encode(prompt, use_tqdm=False)
168 + print(
169 + "################ Inference done (it took seconds) ##############"
170 + )
171 +
172 + return outputs[0].outputs.data
173 +
174 +
175 +def generate_datamodule():
176 + datamodule = Sen1Floods11NonGeoDataModule(
177 + data_root=datamodule_config['data_root'],
178 + batch_size=datamodule_config["batch_size"],
179 + num_workers=datamodule_config["num_workers"],
180 + bands=datamodule_config["bands"],
181 + drop_last=datamodule_config["drop_last"],
182 + test_transform=datamodule_config["test_transform"
183 + ""])
184 +
185 + return datamodule
186 +
187 +
188 +def process_channel_group(orig_img, channels):
189 + """
190 + Args:
191 + orig_img: torch.Tensor representing original image (reference)
192 + with shape = (bands, H, W).
193 + channels: list of indices representing RGB channels.
194 +
195 + Returns:
196 + torch.Tensor with shape (num_channels, height, width) for original image
197 + """
198 +
199 + orig_img = orig_img[channels, ...]
200 + valid_mask = torch.ones_like(orig_img, dtype=torch.bool)
201 + valid_mask[orig_img == NO_DATA_FLOAT] = False
202 +
203 + # Rescale (enhancing contrast)
204 + max_value = max(3000, np.percentile(orig_img[valid_mask], PERCENTILE))
205 + min_value = OFFSET
206 +
207 + orig_img = torch.clamp((orig_img - min_value) / (max_value - min_value), 0,
208 + 1)
209 +
210 + # No data as zeros
211 + orig_img[~valid_mask] = 0
212 +
213 + return orig_img
214 +
215 +
216 +def read_geotiff(file_path: str):
217 + """Read all bands from *file_path* and return image + meta info.
218 +
219 + Args:
220 + file_path: path to image file.
221 +
222 + Returns:
223 + np.ndarray with shape (bands, height, width)
224 + meta info dict
225 + """
226 +
227 + with rasterio.open(file_path) as src:
228 + img = src.read()
229 + meta = src.meta
230 + try:
231 + coords = src.lnglat()
232 + except Exception:
233 + # Cannot read coords
234 + coords = None
235 +
236 + return img, meta, coords
237 +
238 +
239 +def save_geotiff(image, output_path: str, meta: dict):
240 + """Save multi-band image in Geotiff file.
241 +
242 + Args:
243 + image: np.ndarray with shape (bands, height, width)
244 + output_path: path where to save the image
245 + meta: dict with meta info.
246 + """
247 +
248 + with rasterio.open(output_path, "w", **meta) as dest:
249 + for i in range(image.shape[0]):
250 + dest.write(image[i, :, :], i + 1)
251 +
252 + return
253 +
254 +
255 +def _convert_np_uint8(float_image: torch.Tensor):
256 + image = float_image.numpy() * 255.0
257 + image = image.astype(dtype=np.uint8)
258 +
259 + return image
260 +
261 +
262 +def load_example(
263 + file_paths: List[str],
264 + mean: List[float] = None,
265 + std: List[float] = None,
266 + indices: Union[list[int], None] = None,
267 +):
268 + """Build an input example by loading images in *file_paths*.
269 +
270 + Args:
271 + file_paths: list of file paths .
272 + mean: list containing mean values for each band in the images
273 + in *file_paths*.
274 + std: list containing std values for each band in the images
275 + in *file_paths*.
276 +
277 + Returns:
278 + np.array containing created example
279 + list of meta info for each image in *file_paths*
280 + """
281 +
282 + imgs = []
283 + metas = []
284 + temporal_coords = []
285 + location_coords = []
286 +
287 + for file in file_paths:
288 + img, meta, coords = read_geotiff(file)
289 +
290 + # Rescaling (don't normalize on nodata)
291 + img = np.moveaxis(img, 0, -1) # channels last for rescaling
292 + if indices is not None:
293 + img = img[..., indices]
294 + if mean is not None and std is not None:
295 + img = np.where(img == NO_DATA, NO_DATA_FLOAT, (img - mean) / std)
296 +
297 + imgs.append(img)
298 + metas.append(meta)
299 + if coords is not None:
300 + location_coords.append(coords)
301 +
302 + try:
303 + match = re.search(r'(\d{7,8}T\d{6})', file)
304 + if match:
305 + year = int(match.group(1)[:4])
306 + julian_day = match.group(1).split('T')[0][4:]
307 + if len(julian_day) == 3:
308 + julian_day = int(julian_day)
309 + else:
310 + julian_day = datetime.datetime.strptime(
311 + julian_day, '%m%d').timetuple().tm_yday
312 + temporal_coords.append([year, julian_day])
313 + except Exception as e:
314 + print(f'Could not extract timestamp for {file} ({e})')
315 +
316 + imgs = np.stack(imgs, axis=0) # num_frames, H, W, C
317 + imgs = np.moveaxis(imgs, -1, 0).astype("float32")
318 + imgs = np.expand_dims(imgs, axis=0) # add batch di
319 +
320 + return imgs, temporal_coords, location_coords, metas
321 +
322 +
323 +def run_model(input_data,
324 + temporal_coords,
325 + location_coords,
326 + model,
327 + datamodule,
328 + img_size,
329 + lightning_model=None):
330 + # Reflect pad if not divisible by img_size
331 + original_h, original_w = input_data.shape[-2:]
332 + pad_h = (img_size - (original_h % img_size)) % img_size
333 + pad_w = (img_size - (original_w % img_size)) % img_size
334 + input_data = np.pad(input_data,
335 + ((0, 0), (0, 0), (0, 0), (0, pad_h), (0, pad_w)),
336 + mode="reflect")
337 +
338 + # Build sliding window
339 + batch_size = 1
340 + batch = torch.tensor(input_data, device="cpu")
341 + windows = (batch.unfold(3, img_size,
342 + img_size).unfold(4, img_size, img_size))
343 + h1, w1 = windows.shape[3:5]
344 + windows = rearrange(windows,
345 + "b c t h1 w1 h w -> (b h1 w1) c t h w",
346 + h=img_size,
347 + w=img_size)
348 +
349 + # Split into batches if number of windows > batch_size
350 + num_batches = windows.shape[0] // batch_size if windows.shape[
351 + 0] > batch_size else 1
352 + windows = torch.tensor_split(windows, num_batches, dim=0)
353 +
354 + if torch.cuda.is_available():
355 + device = torch.device('cuda')
356 + else:
357 + device = torch.device('cpu')
358 +
359 + if temporal_coords:
360 + temporal_coords = torch.tensor(temporal_coords,
361 + device=device).unsqueeze(0)
362 + else:
363 + temporal_coords = None
364 + if location_coords:
365 + location_coords = torch.tensor(location_coords[0],
366 + device=device).unsqueeze(0)
367 + else:
368 + location_coords = None
369 +
370 + # Run model
371 + pred_imgs = []
372 + for x in windows:
373 + # Apply standardization
374 + x = datamodule.test_transform(
375 + image=x.squeeze().numpy().transpose(1, 2, 0))
376 + x = datamodule.aug(x)['image']
377 +
378 + with torch.no_grad():
379 + x = x.to(device)
380 + pred = model.run(x, location_coords=location_coords)
381 + if lightning_model:
382 + pred_lightning = lightning_model(
383 + x,
384 + temporal_coords=temporal_coords,
385 + location_coords=location_coords)
386 + pred_lightning = pred_lightning.output.detach().cpu()
387 + if not torch.equal(pred, pred_lightning):
388 + print("Inference output is not equal")
389 + y_hat = pred.argmax(dim=1)
390 +
391 + y_hat = torch.nn.functional.interpolate(y_hat.unsqueeze(1).float(),
392 + size=img_size,
393 + mode="nearest")
394 +
395 + pred_imgs.append(y_hat)
396 +
397 + pred_imgs = torch.concat(pred_imgs, dim=0)
398 +
399 + # Build images from patches
400 + pred_imgs = rearrange(
401 + pred_imgs,
402 + "(b h1 w1) c h w -> b c (h1 h) (w1 w)",
403 + h=img_size,
404 + w=img_size,
405 + b=1,
406 + c=1,
407 + h1=h1,
408 + w1=w1,
409 + )
410 +
411 + # Cut padded area back to original size
412 + pred_imgs = pred_imgs[..., :original_h, :original_w]
413 +
414 + # Squeeze (batch size 1)
415 + pred_imgs = pred_imgs[0]
416 +
417 + return pred_imgs
418 +
419 +
420 +def main(
421 + data_file: str,
422 + output_dir: str,
423 + rgb_outputs: bool,
424 + input_indices: list[int] = None,
425 +):
426 + os.makedirs(output_dir, exist_ok=True)
427 +
428 + # Load model ---------------------------------------------------------------
429 +
430 + model_obj = PrithviMAE()
431 + datamodule = generate_datamodule()
432 + img_size = 256 # Size of Sen1Floods11
433 +
434 + # Loading data -------------------------------------------------------------
435 +
436 + input_data, temporal_coords, location_coords, meta_data = load_example(
437 + file_paths=[data_file],
438 + indices=input_indices,
439 + )
440 +
441 + meta_data = meta_data[0] # only one image
442 +
443 + if input_data.mean() > 1:
444 + input_data = input_data / 10000 # Convert to range 0-1
445 +
446 + # Running model ------------------------------------------------------------
447 +
448 + channels = [
449 + datamodule_config['bands'].index(b) for b in ["RED", "GREEN", "BLUE"]
450 + ] # BGR -> RGB
451 +
452 + pred = run_model(input_data, temporal_coords, location_coords, model_obj,
453 + datamodule, img_size)
454 +
455 + # Save pred
456 + meta_data.update(count=1, dtype="uint8", compress="lzw", nodata=0)
457 + pred_file = os.path.join(
458 + output_dir,
459 + f"pred_{os.path.splitext(os.path.basename(data_file))[0]}.tiff")
460 + save_geotiff(_convert_np_uint8(pred), pred_file, meta_data)
461 +
462 + # Save image + pred
463 + meta_data.update(count=3, dtype="uint8", compress="lzw", nodata=0)
464 +
465 + if input_data.mean() < 1:
466 + input_data = input_data * 10000 # Scale to 0-10000
467 +
468 + rgb_orig = process_channel_group(
469 + orig_img=torch.Tensor(input_data[0, :, 0, ...]),
470 + channels=channels,
471 + )
472 +
473 + pred[pred == 0.] = np.nan
474 + img_pred = rgb_orig * 0.7 + pred * 0.3
475 + img_pred[img_pred.isnan()] = rgb_orig[img_pred.isnan()]
476 +
477 + img_pred_file = os.path.join(
478 + output_dir,
479 + f"rgb_pred_{os.path.splitext(os.path.basename(data_file))[0]}.tiff")
480 + save_geotiff(
481 + image=_convert_np_uint8(img_pred),
482 + output_path=img_pred_file,
483 + meta=meta_data,
484 + )
485 +
486 + # Save image rgb
487 + if rgb_outputs:
488 + rgb_file = os.path.join(
489 + output_dir, "original_rgb_"
490 + f"{os.path.splitext(os.path.basename(data_file))[0]}.tiff")
491 + save_geotiff(
492 + image=_convert_np_uint8(rgb_orig),
493 + output_path=rgb_file,
494 + meta=meta_data,
495 + )
496 +
497 +
498 +if __name__ == "__main__":
499 + parser = argparse.ArgumentParser("MAE run inference", add_help=False)
500 +
501 + parser.add_argument(
502 + "--data_file",
503 + type=str,
504 + default="./India_900498_S2Hand.tif",
505 + help="Path to the file.",
506 + )
507 + parser.add_argument(
508 + "--output_dir",
509 + type=str,
510 + default="output",
511 + help="Path to the directory where to save outputs.",
512 + )
513 + parser.add_argument(
514 + "--input_indices",
515 + default=[1, 2, 3, 8, 11, 12],
516 + type=int,
517 + nargs="+",
518 + help=
519 + "0-based indices of the six Prithvi channels to be selected from the "
520 + "input. By default selects [1,2,3,8,11,12] for S2L1C data.",
521 + )
522 + parser.add_argument(
523 + "--rgb_outputs",
524 + action="store_true",
525 + help="If present, output files will only contain RGB channels. "
526 + "Otherwise, all bands will be saved.",
527 + )
528 + args = parser.parse_args()
529 +
530 + main(**vars(args))
tests/models/registry.py
+4
@@ -214,6 +214,10 @@ _EMBEDDING_EXAMPLE_MODELS = {
214 "Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
215 trust_remote_code=True),
216 "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"), # noqa: E501
217 + # The model on Huggingface is currently being updated,
218 + # hence I temporarily mark it as not available online
219 + "PrithviGeoSpatialMAE": _HfExamplesInfo("ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", # noqa: E501
220 + is_available_online=False),
221 }
222
223 _CROSS_ENCODER_EXAMPLE_MODELS = {
vllm/attention/backends/placeholder_attn.py
+8 -3
@@ -320,9 +320,14 @@ class PlaceholderAttentionMetadataBuilder(
320 -1 if cuda graph is not used.
321 batch_size: The maybe padded batch size.
322 """
323 - for inter_data in self.input_builder.inter_data_list:
324 - self._add_seq_group(inter_data,
325 - self.input_builder.chunked_prefill_enabled)
323 +
324 + # Some input builders such as ModelInputForCPUBuilder do not have the
325 + # "inter_data_list" attribute.
326 + # Let's check inter_data_list exists before we reference it.
327 + if hasattr(self.input_builder, "inter_data_list"):
328 + for inter_data in self.input_builder.inter_data_list:
329 + self._add_seq_group(inter_data,
330 + self.input_builder.chunked_prefill_enabled)
331
332 device = self.runner.device
333 use_captured_graph = cuda_graph_pad_size != -1
vllm/inputs/preprocess.py
+17 -5
@@ -254,8 +254,14 @@ class InputPreprocessor:
254 Apply the model's multi-modal processor to a multi-modal prompt,
255 returning the corresponding token IDs and metadata.
256 """
257 - tokenizer_group = self.get_tokenizer_group()
258 - tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
257 + # At the moment on model (PrithviGeoSpatialMAE) requires to be
258 + # initialized without a tokenizer while using also multi-modal
259 + # input.
260 + if not self.tokenizer:
261 + tokenizer = None
262 + else:
263 + tokenizer_group = self.get_tokenizer_group()
264 + tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
265
266 mm_processor = self.mm_registry.create_processor(
267 self.model_config, tokenizer)
@@ -273,9 +279,15 @@ class InputPreprocessor:
279 lora_request: Optional[LoRARequest],
280 ) -> MultiModalInputs:
281 """Async version of :meth:`_process_multimodal`."""
276 - tokenizer_group = self.get_tokenizer_group()
277 - tokenizer = await tokenizer_group.get_lora_tokenizer_async(lora_request
278 - )
282 + # At the moment on model (PrithviGeoSpatialMAE) requires to be
283 + # initialized without a tokenizer while using also multi-modal
284 + # input.
285 + if not self.tokenizer:
286 + tokenizer = None
287 + else:
288 + tokenizer_group = self.get_tokenizer_group()
289 + tokenizer = await tokenizer_group.get_lora_tokenizer_async(
290 + lora_request)
291
292 mm_processor = self.mm_registry.create_processor(
293 self.model_config, tokenizer)
vllm/model_executor/models/prithvi_geospatial_mae.py new
+238
@@ -0,0 +1,238 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +# Copyright 2025 The vLLM team.
4 +# Copyright 2025 IBM.
5 +#
6 +# Licensed under the Apache License, Version 2.0 (the "License");
7 +# you may not use this file except in compliance with the License.
8 +# You may obtain a copy of the License at
9 +#
10 +# http://www.apache.org/licenses/LICENSE-2.0
11 +#
12 +# Unless required by applicable law or agreed to in writing, software
13 +# distributed under the License is distributed on an "AS IS" BASIS,
14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 +# See the License for the specific language governing permissions and
16 +# limitations under the License.
17 +"""Inference-only IBM/NASA Prithvi Geospatial model."""
18 +from typing import Iterable, List, Mapping, Optional, Set, Tuple, Union
19 +
20 +import torch
21 +import torch.nn as nn
22 +from transformers import BatchFeature
23 +
24 +from vllm.attention import AttentionMetadata
25 +from vllm.config import VllmConfig
26 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 +from vllm.model_executor.models.interfaces import (IsAttentionFree,
28 + SupportsMultiModal)
29 +from vllm.model_executor.models.utils import AutoWeightsLoader
30 +from vllm.model_executor.pooling_metadata import PoolingMetadata
31 +from vllm.multimodal import MULTIMODAL_REGISTRY
32 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
33 + MultiModalInputs, MultiModalKwargs)
34 +from vllm.multimodal.parse import MultiModalDataItems
35 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
36 + BaseProcessingInfo, PromptReplacement)
37 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38 +from vllm.sequence import (IntermediateTensors, PoolerOutput,
39 + PoolingSequenceGroupOutput)
40 +
41 +
42 +class PrithviGeoSpatialMAEProcessingInfo(BaseProcessingInfo):
43 +
44 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
45 + return {"image": None}
46 +
47 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
48 + pass
49 +
50 +
51 +class PrithviGeoSpatialMAEInputBuilder(
52 + BaseDummyInputsBuilder[PrithviGeoSpatialMAEProcessingInfo]):
53 +
54 + def get_dummy_processor_inputs(
55 + self,
56 + seq_len: int,
57 + mm_counts: Mapping[str, int],
58 + ) -> ProcessorInputs:
59 + return ProcessorInputs(
60 + prompt_text="",
61 + # This model input is fixed and is in the form of a torch Tensor.
62 + # The size of pixel_values might change in the cases where we resize
63 + # the input but never exceeds the dimensions below.
64 + mm_data={
65 + "pixel_values": torch.full((1, 6, 512, 512), 1.0),
66 + "location_coords": torch.full((1, 2), 1.0)
67 + })
68 +
69 +
70 +class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
71 +
72 + def _get_mm_fields_config(
73 + self,
74 + hf_inputs: BatchFeature,
75 + hf_processor_mm_kwargs: Mapping[str, object],
76 + ) -> Mapping[str, MultiModalFieldConfig]:
77 + return dict(
78 + pixel_values=MultiModalFieldConfig.batched("image"),
79 + location_coords=MultiModalFieldConfig.batched("image"),
80 + )
81 +
82 + def _get_prompt_replacements(
83 + self,
84 + mm_items: MultiModalDataItems,
85 + hf_processor_mm_kwargs: Mapping[str, object],
86 + out_mm_kwargs: MultiModalKwargs,
87 + ) -> list[PromptReplacement]:
88 + pass
89 +
90 + def _get_mm_fields_config(
91 + self,
92 + hf_inputs: BatchFeature,
93 + hf_processor_mm_kwargs: Mapping[str, object],
94 + ) -> Mapping[str, MultiModalFieldConfig]:
95 + pass
96 +
97 + def apply(
98 + self,
99 + prompt: Union[str, list[int]],
100 + mm_data: MultiModalDataDict,
101 + hf_processor_mm_kwargs: Mapping[str, object],
102 + ) -> MultiModalInputs:
103 + mm_kwargs = {}
104 +
105 + for k, v in mm_data.items():
106 + mm_kwargs[k] = v
107 +
108 + return MultiModalInputs(
109 + type="multimodal",
110 + prompt=prompt,
111 + prompt_token_ids=[1],
112 + mm_kwargs=MultiModalKwargs(mm_kwargs),
113 + mm_placeholders={},
114 + )
115 +
116 +
117 +@MULTIMODAL_REGISTRY.register_processor(
118 + PrithviGeoSpatialMAEMultiModalProcessor,
119 + info=PrithviGeoSpatialMAEProcessingInfo,
120 + dummy_inputs=PrithviGeoSpatialMAEInputBuilder)
121 +class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal):
122 + """ Prithvi Masked Autoencoder"""
123 +
124 + def _instantiate_model(self, config: dict) -> nn.Module | None:
125 +
126 + # We might be able/need to support different tasks with this same model
127 + if config["task_args"]["task"] == "SemanticSegmentationTask":
128 + from terratorch.cli_tools import SemanticSegmentationTask
129 + task = SemanticSegmentationTask(
130 + config["model_args"],
131 + config["task_args"]["model_factory"],
132 + loss=config["task_args"]["loss"],
133 + lr=config["task_args"]["lr"],
134 + ignore_index=config["task_args"]["ignore_index"],
135 + optimizer=config["task_args"]["optimizer"],
136 + optimizer_hparams=config["optimizer_params"],
137 + scheduler=config["task_args"]["scheduler"],
138 + scheduler_hparams=config["scheduler_params"],
139 + plot_on_val=config["task_args"]["plot_on_val"],
140 + freeze_decoder=config["task_args"]["freeze_decoder"],
141 + freeze_backbone=config["task_args"]["freeze_backbone"])
142 +
143 + return task.model
144 + else:
145 + return None
146 +
147 + def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
148 + super().__init__()
149 +
150 + # the actual model is dynamically instantiated using terratorch
151 + # allowing us to perform changes to the model architecture
152 + # at startup time (e.g., change the model decoder class.)
153 + self.model = self._instantiate_model(
154 + vllm_config.model_config.hf_config.to_dict()["pretrained_cfg"])
155 + if self.model is None:
156 + raise ValueError(
157 + "Unsupported task."
158 + "Only SemanticSegmentationTask is supported for now"
159 + "by PrithviGeospatialMAE.")
160 +
161 + def _parse_and_validate_multimodal_data(
162 + self, **kwargs) -> Tuple[torch.Tensor, torch.Tensor | None]:
163 +
164 + pixel_values = kwargs.pop("pixel_values", None)
165 + if not isinstance(pixel_values, torch.Tensor):
166 + raise ValueError(f"Incorrect type of pixel_values. "
167 + f"Got type: {type(pixel_values)}")
168 + pixel_values = torch.unbind(pixel_values, dim=0)[0]
169 +
170 + location_coords = kwargs.pop("location_coords", None)
171 + if not isinstance(location_coords, torch.Tensor):
172 + raise ValueError(f"Incorrect type of location_coords. "
173 + f"Got type: {type(location_coords)}")
174 + location_coords = torch.unbind(location_coords, dim=0)[0]
175 + if location_coords.shape == torch.Size([0]):
176 + location_coords = None
177 +
178 + return pixel_values, location_coords
179 +
180 + def forward(
181 + self,
182 + input_ids: Optional[torch.Tensor],
183 + positions: torch.Tensor,
184 + kv_caches: List[torch.Tensor],
185 + attn_metadata: AttentionMetadata,
186 + intermediate_tensors: Optional[IntermediateTensors] = None,
187 + inputs_embeds: Optional[torch.Tensor] = None,
188 + **kwargs: object,
189 + ):
190 +
191 + pixel_values, location_coords = (
192 + self._parse_and_validate_multimodal_data(**kwargs))
193 + model_output = self.model(pixel_values,
194 + location_coords=location_coords)
195 +
196 + return model_output.output
197 +
198 + def pooler(
199 + self,
200 + hidden_states: torch.Tensor,
201 + pooling_metadata: PoolingMetadata,
202 + ) -> Optional[PoolerOutput]:
203 + return PoolerOutput([PoolingSequenceGroupOutput(hidden_states)])
204 +
205 + def load_weights(self, weights: Iterable[Tuple[str,
206 + torch.Tensor]]) -> Set[str]:
207 + params_list = []
208 + model_buffers = dict(self.named_buffers())
209 + loaded_buffers = []
210 + for key, value in weights:
211 + if key == "state_dict":
212 + weights_to_parse = value
213 + for name, weight in weights_to_parse.items():
214 + if "pos_embed" in name:
215 + continue
216 +
217 + if "_timm_module." in name:
218 + name = name.replace("_timm_module.", "")
219 +
220 + # this model requires a couple of buffers to be loaded
221 + # that are not loadable with the AutoWeightsLoader
222 + if name in model_buffers:
223 + if "_timm_module." in name:
224 + name = name.replace("_timm_module.", "")
225 + buffer = model_buffers[name]
226 + weight_loader = getattr(buffer, "weight_loader",
227 + default_weight_loader)
228 + weight_loader(buffer, weight)
229 + loaded_buffers.append(name)
230 + else:
231 + params_list.append((name, weight))
232 + break
233 +
234 + # Load the remaining model parameters
235 + loader = AutoWeightsLoader(self)
236 + autoloaded_weights = loader.load_weights(params_list)
237 +
238 + return autoloaded_weights.union(set(loaded_buffers))
vllm/model_executor/models/registry.py
+4
@@ -137,6 +137,10 @@ _EMBEDDING_MODELS = {
137 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
138 # [Auto-converted (see adapters.py)]
139 "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
140 + # Technically PrithviGeoSpatialMAE is a model that works on images, both in
141 + # input and output. I am adding it here because it piggy-backs on embedding
142 + # models for the time being.
143 + "PrithviGeoSpatialMAE": ("prithvi_geospatial_mae", "PrithviGeoSpatialMAE"),
144 }
145
146 _CROSS_ENCODER_MODELS = {
vllm/worker/pooling_model_runner.py
+10 -1
@@ -74,7 +74,16 @@ class PoolingModelRunner(
74 prefill_meta = model_input.attn_metadata.prefill_metadata
75 decode_meta = model_input.attn_metadata.decode_metadata
76 virtual_engine = model_input.virtual_engine
77 - if prefill_meta is None and decode_meta.use_cuda_graph:
77 + # Pooling models are (ab-)used also to integrate non text models that
78 + # are not autoregressive (PrithviGeosaptialMAE).
79 + # These model might not use attention and do not really have a prefill
80 + # and decode phase. The model input is processed in one shot and both
81 + # decode_metadata and prefill_metadata would be None for such models.
82 + # See the PlaceholderAttentionMetadata class.
83 + # TODO: Figure out if cuda_graph is of any use for these models and
84 + # explore how to leverage it.
85 + if (prefill_meta is None and decode_meta is not None
86 + and decode_meta.use_cuda_graph):
87 assert model_input.input_tokens is not None
88 graph_batch_size = model_input.input_tokens.shape[0]
89 model_executable = self.graph_runners[virtual_engine][