A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Model] IBM/NASA Prithvi Geospatial model (#12830)
Christian Pinto committed
Feb 12, 2025 at 04:34 UTC
974dfd497149e871e59e35b677a85cca66ec3bae
7 files changed
+811
-9
examples/offline_inference/prithvi_geospatial_mae.py
new
+530
@@ -0,0 +1,530 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""
3
+This is a demo script showing how to use the
4
+PrithviGeospatialMAE model with vLLM
5
+This script is based on: https://huggingface.co/ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11/blob/main/inference.py # noqa
6
+
7
+Target model weights: https://huggingface.co/ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11/resolve/main/Prithvi-EO-V2-300M-TL-Sen1Floods11.pt # noqa
8
+
9
+The requirements for running this script are:
10
+- Installing [terratorch, albumentations, rasterio] in your python environment
11
+- downloading the model weights in a 'model' folder local to the script
12
+ (temporary measure until the proper config.json file is uploaded to HF)
13
+- download an input example image (India_900498_S2Hand.tif) and place it in
14
+ the same folder with the script (or specify with the --data_file argument)
15
+
16
+Run the example:
17
+python prithvi_geospatial_mae.py
18
+
19
+""" # noqa: E501
20
+import argparse
21
+import datetime
22
+import os
23
+import re
24
+from typing import List, Union
25
+
26
+import albumentations
27
+import numpy as np
28
+import rasterio
29
+import torch
30
+from einops import rearrange
31
+from terratorch.datamodules import Sen1Floods11NonGeoDataModule
32
+
33
+from vllm import LLM
34
+
35
+NO_DATA = -9999
36
+NO_DATA_FLOAT = 0.0001
37
+OFFSET = 0
38
+PERCENTILE = 99
39
+
40
+model_config = """{
41
+ "architectures": ["PrithviGeoSpatialMAE"],
42
+ "num_classes": 0,
43
+ "pretrained_cfg": {
44
+ "task_args": {
45
+ "task": "SemanticSegmentationTask",
46
+ "model_factory": "EncoderDecoderFactory",
47
+ "loss": "ce",
48
+ "ignore_index": -1,
49
+ "lr": 0.001,
50
+ "freeze_backbone": false,
51
+ "freeze_decoder": false,
52
+ "plot_on_val": 10,
53
+ "optimizer": "AdamW",
54
+ "scheduler": "CosineAnnealingLR"
55
+ },
56
+ "model_args": {
57
+ "backbone_pretrained": false,
58
+ "backbone": "prithvi_eo_v2_300_tl",
59
+ "decoder": "UperNetDecoder",
60
+ "decoder_channels": 256,
61
+ "decoder_scale_modules": true,
62
+ "num_classes": 2,
63
+ "rescale": true,
64
+ "backbone_bands": [
65
+ "BLUE",
66
+ "GREEN",
67
+ "RED",
68
+ "NIR_NARROW",
69
+ "SWIR_1",
70
+ "SWIR_2"
71
+ ],
72
+ "head_dropout": 0.1,
73
+ "necks": [
74
+ {
75
+ "name": "SelectIndices",
76
+ "indices": [
77
+ 5,
78
+ 11,
79
+ 17,
80
+ 23
81
+ ]
82
+ },
83
+ {
84
+ "name": "ReshapeTokensToImage"
85
+ }
86
+ ]
87
+ },
88
+ "optimizer_params" : {
89
+ "lr": 5.0e-05,
90
+ "betas": [0.9, 0.999],
91
+ "eps": [1.0e-08],
92
+ "weight_decay": 0.05,
93
+ "amsgrad": false,
94
+ "maximize": false,
95
+ "capturable": false,
96
+ "differentiable": false
97
+ },
98
+ "scheduler_params" : {
99
+ "T_max": 50,
100
+ "eta_min": 0,
101
+ "last_epoch": -1,
102
+ "verbose": "deprecated"
103
+ }
104
+ },
105
+
106
+
107
+ "torch_dtype": "float32"
108
+}
109
+"""
110
+
111
+# Temporarily creating the "config.json" for the model.
112
+# This is going to disappear once the correct config.json is available on HF
113
+with open(os.path.join(os.path.dirname(__file__), "./model/config.json"),
114
+ 'w') as config_file:
115
+ config_file.write(model_config)
116
+
117
+datamodule_config = {
118
+ 'bands': ['BLUE', 'GREEN', 'RED', 'NIR_NARROW', 'SWIR_1', 'SWIR_2'],
119
+ 'batch_size':
120
+ 16,
121
+ 'constant_scale':
122
+ 0.0001,
123
+ 'data_root':
124
+ '/dccstor/geofm-finetuning/datasets/sen1floods11',
125
+ 'drop_last':
126
+ True,
127
+ 'no_data_replace':
128
+ 0.0,
129
+ 'no_label_replace':
130
+ -1,
131
+ 'num_workers':
132
+ 8,
133
+ 'test_transform': [
134
+ albumentations.Resize(always_apply=False,
135
+ height=448,
136
+ interpolation=1,
137
+ p=1,
138
+ width=448),
139
+ albumentations.pytorch.ToTensorV2(transpose_mask=False,
140
+ always_apply=True,
141
+ p=1.0)
142
+ ],
143
+}
144
+
145
+
146
+class PrithviMAE:
147
+
148
+ def __init__(self):
149
+ print("Initializing PrithviMAE model")
150
+ self.model = LLM(model=os.path.join(os.path.dirname(__file__),
151
+ "./model"),
152
+ skip_tokenizer_init=True,
153
+ dtype="float32")
154
+
155
+ def run(self, input_data, location_coords):
156
+ print("################ Running inference on vLLM ##############")
157
+ # merge the inputs into one data structure
158
+ mm_data = {
159
+ "pixel_values":
160
+ torch.empty(0) if input_data is None else input_data,
161
+ "location_coords":
162
+ torch.empty(0) if location_coords is None else location_coords
163
+ }
164
+
165
+ prompt = {"prompt_token_ids": [1], "multi_modal_data": mm_data}
166
+
167
+ outputs = self.model.encode(prompt, use_tqdm=False)
168
+ print(
169
+ "################ Inference done (it took seconds) ##############"
170
+ )
171
+
172
+ return outputs[0].outputs.data
173
+
174
+
175
+def generate_datamodule():
176
+ datamodule = Sen1Floods11NonGeoDataModule(
177
+ data_root=datamodule_config['data_root'],
178
+ batch_size=datamodule_config["batch_size"],
179
+ num_workers=datamodule_config["num_workers"],
180
+ bands=datamodule_config["bands"],
181
+ drop_last=datamodule_config["drop_last"],
182
+ test_transform=datamodule_config["test_transform"
183
+ ""])
184
+
185
+ return datamodule
186
+
187
+
188
+def process_channel_group(orig_img, channels):
189
+ """
190
+ Args:
191
+ orig_img: torch.Tensor representing original image (reference)
192
+ with shape = (bands, H, W).
193
+ channels: list of indices representing RGB channels.
194
+
195
+ Returns:
196
+ torch.Tensor with shape (num_channels, height, width) for original image
197
+ """
198
+
199
+ orig_img = orig_img[channels, ...]
200
+ valid_mask = torch.ones_like(orig_img, dtype=torch.bool)
201
+ valid_mask[orig_img == NO_DATA_FLOAT] = False
202
+
203
+ # Rescale (enhancing contrast)
204
+ max_value = max(3000, np.percentile(orig_img[valid_mask], PERCENTILE))
205
+ min_value = OFFSET
206
+
207
+ orig_img = torch.clamp((orig_img - min_value) / (max_value - min_value), 0,
208
+ 1)
209
+
210
+ # No data as zeros
211
+ orig_img[~valid_mask] = 0
212
+
213
+ return orig_img
214
+
215
+
216
+def read_geotiff(file_path: str):
217
+ """Read all bands from *file_path* and return image + meta info.
218
+
219
+ Args:
220
+ file_path: path to image file.
221
+
222
+ Returns:
223
+ np.ndarray with shape (bands, height, width)
224
+ meta info dict
225
+ """
226
+
227
+ with rasterio.open(file_path) as src:
228
+ img = src.read()
229
+ meta = src.meta
230
+ try:
231
+ coords = src.lnglat()
232
+ except Exception:
233
+ # Cannot read coords
234
+ coords = None
235
+
236
+ return img, meta, coords
237
+
238
+
239
+def save_geotiff(image, output_path: str, meta: dict):
240
+ """Save multi-band image in Geotiff file.
241
+
242
+ Args:
243
+ image: np.ndarray with shape (bands, height, width)
244
+ output_path: path where to save the image
245
+ meta: dict with meta info.
246
+ """
247
+
248
+ with rasterio.open(output_path, "w", **meta) as dest:
249
+ for i in range(image.shape[0]):
250
+ dest.write(image[i, :, :], i + 1)
251
+
252
+ return
253
+
254
+
255
+def _convert_np_uint8(float_image: torch.Tensor):
256
+ image = float_image.numpy() * 255.0
257
+ image = image.astype(dtype=np.uint8)
258
+
259
+ return image
260
+
261
+
262
+def load_example(
263
+ file_paths: List[str],
264
+ mean: List[float] = None,
265
+ std: List[float] = None,
266
+ indices: Union[list[int], None] = None,
267
+):
268
+ """Build an input example by loading images in *file_paths*.
269
+
270
+ Args:
271
+ file_paths: list of file paths .
272
+ mean: list containing mean values for each band in the images
273
+ in *file_paths*.
274
+ std: list containing std values for each band in the images
275
+ in *file_paths*.
276
+
277
+ Returns:
278
+ np.array containing created example
279
+ list of meta info for each image in *file_paths*
280
+ """
281
+
282
+ imgs = []
283
+ metas = []
284
+ temporal_coords = []
285
+ location_coords = []
286
+
287
+ for file in file_paths:
288
+ img, meta, coords = read_geotiff(file)
289
+
290
+ # Rescaling (don't normalize on nodata)
291
+ img = np.moveaxis(img, 0, -1) # channels last for rescaling
292
+ if indices is not None:
293
+ img = img[..., indices]
294
+ if mean is not None and std is not None:
295
+ img = np.where(img == NO_DATA, NO_DATA_FLOAT, (img - mean) / std)
296
+
297
+ imgs.append(img)
298
+ metas.append(meta)
299
+ if coords is not None:
300
+ location_coords.append(coords)
301
+
302
+ try:
303
+ match = re.search(r'(\d{7,8}T\d{6})', file)
304
+ if match:
305
+ year = int(match.group(1)[:4])
306
+ julian_day = match.group(1).split('T')[0][4:]
307
+ if len(julian_day) == 3:
308
+ julian_day = int(julian_day)
309
+ else:
310
+ julian_day = datetime.datetime.strptime(
311
+ julian_day, '%m%d').timetuple().tm_yday
312
+ temporal_coords.append([year, julian_day])
313
+ except Exception as e:
314
+ print(f'Could not extract timestamp for {file} ({e})')
315
+
316
+ imgs = np.stack(imgs, axis=0) # num_frames, H, W, C
317
+ imgs = np.moveaxis(imgs, -1, 0).astype("float32")
318
+ imgs = np.expand_dims(imgs, axis=0) # add batch di
319
+
320
+ return imgs, temporal_coords, location_coords, metas
321
+
322
+
323
+def run_model(input_data,
324
+ temporal_coords,
325
+ location_coords,
326
+ model,
327
+ datamodule,
328
+ img_size,
329
+ lightning_model=None):
330
+ # Reflect pad if not divisible by img_size
331
+ original_h, original_w = input_data.shape[-2:]
332
+ pad_h = (img_size - (original_h % img_size)) % img_size
333
+ pad_w = (img_size - (original_w % img_size)) % img_size
334
+ input_data = np.pad(input_data,
335
+ ((0, 0), (0, 0), (0, 0), (0, pad_h), (0, pad_w)),
336
+ mode="reflect")
337
+
338
+ # Build sliding window
339
+ batch_size = 1
340
+ batch = torch.tensor(input_data, device="cpu")
341
+ windows = (batch.unfold(3, img_size,
342
+ img_size).unfold(4, img_size, img_size))
343
+ h1, w1 = windows.shape[3:5]
344
+ windows = rearrange(windows,
345
+ "b c t h1 w1 h w -> (b h1 w1) c t h w",
346
+ h=img_size,
347
+ w=img_size)
348
+
349
+ # Split into batches if number of windows > batch_size
350
+ num_batches = windows.shape[0] // batch_size if windows.shape[
351
+ 0] > batch_size else 1
352
+ windows = torch.tensor_split(windows, num_batches, dim=0)
353
+
354
+ if torch.cuda.is_available():
355
+ device = torch.device('cuda')
356
+ else:
357
+ device = torch.device('cpu')
358
+
359
+ if temporal_coords:
360
+ temporal_coords = torch.tensor(temporal_coords,
361
+ device=device).unsqueeze(0)
362
+ else:
363
+ temporal_coords = None
364
+ if location_coords:
365
+ location_coords = torch.tensor(location_coords[0],
366
+ device=device).unsqueeze(0)
367
+ else:
368
+ location_coords = None
369
+
370
+ # Run model
371
+ pred_imgs = []
372
+ for x in windows:
373
+ # Apply standardization
374
+ x = datamodule.test_transform(
375
+ image=x.squeeze().numpy().transpose(1, 2, 0))
376
+ x = datamodule.aug(x)['image']
377
+
378
+ with torch.no_grad():
379
+ x = x.to(device)
380
+ pred = model.run(x, location_coords=location_coords)
381
+ if lightning_model:
382
+ pred_lightning = lightning_model(
383
+ x,
384
+ temporal_coords=temporal_coords,
385
+ location_coords=location_coords)
386
+ pred_lightning = pred_lightning.output.detach().cpu()
387
+ if not torch.equal(pred, pred_lightning):
388
+ print("Inference output is not equal")
389
+ y_hat = pred.argmax(dim=1)
390
+
391
+ y_hat = torch.nn.functional.interpolate(y_hat.unsqueeze(1).float(),
392
+ size=img_size,
393
+ mode="nearest")
394
+
395
+ pred_imgs.append(y_hat)
396
+
397
+ pred_imgs = torch.concat(pred_imgs, dim=0)
398
+
399
+ # Build images from patches
400
+ pred_imgs = rearrange(
401
+ pred_imgs,
402
+ "(b h1 w1) c h w -> b c (h1 h) (w1 w)",
403
+ h=img_size,
404
+ w=img_size,
405
+ b=1,
406
+ c=1,
407
+ h1=h1,
408
+ w1=w1,
409
+ )
410
+
411
+ # Cut padded area back to original size
412
+ pred_imgs = pred_imgs[..., :original_h, :original_w]
413
+
414
+ # Squeeze (batch size 1)
415
+ pred_imgs = pred_imgs[0]
416
+
417
+ return pred_imgs
418
+
419
+
420
+def main(
421
+ data_file: str,
422
+ output_dir: str,
423
+ rgb_outputs: bool,
424
+ input_indices: list[int] = None,
425
+):
426
+ os.makedirs(output_dir, exist_ok=True)
427
+
428
+ # Load model ---------------------------------------------------------------
429
+
430
+ model_obj = PrithviMAE()
431
+ datamodule = generate_datamodule()
432
+ img_size = 256 # Size of Sen1Floods11
433
+
434
+ # Loading data -------------------------------------------------------------
435
+
436
+ input_data, temporal_coords, location_coords, meta_data = load_example(
437
+ file_paths=[data_file],
438
+ indices=input_indices,
439
+ )
440
+
441
+ meta_data = meta_data[0] # only one image
442
+
443
+ if input_data.mean() > 1:
444
+ input_data = input_data / 10000 # Convert to range 0-1
445
+
446
+ # Running model ------------------------------------------------------------
447
+
448
+ channels = [
449
+ datamodule_config['bands'].index(b) for b in ["RED", "GREEN", "BLUE"]
450
+ ] # BGR -> RGB
451
+
452
+ pred = run_model(input_data, temporal_coords, location_coords, model_obj,
453
+ datamodule, img_size)
454
+
455
+ # Save pred
456
+ meta_data.update(count=1, dtype="uint8", compress="lzw", nodata=0)
457
+ pred_file = os.path.join(
458
+ output_dir,
459
+ f"pred_{os.path.splitext(os.path.basename(data_file))[0]}.tiff")
460
+ save_geotiff(_convert_np_uint8(pred), pred_file, meta_data)
461
+
462
+ # Save image + pred
463
+ meta_data.update(count=3, dtype="uint8", compress="lzw", nodata=0)
464
+
465
+ if input_data.mean() < 1:
466
+ input_data = input_data * 10000 # Scale to 0-10000
467
+
468
+ rgb_orig = process_channel_group(
469
+ orig_img=torch.Tensor(input_data[0, :, 0, ...]),
470
+ channels=channels,
471
+ )
472
+
473
+ pred[pred == 0.] = np.nan
474
+ img_pred = rgb_orig * 0.7 + pred * 0.3
475
+ img_pred[img_pred.isnan()] = rgb_orig[img_pred.isnan()]
476
+
477
+ img_pred_file = os.path.join(
478
+ output_dir,
479
+ f"rgb_pred_{os.path.splitext(os.path.basename(data_file))[0]}.tiff")
480
+ save_geotiff(
481
+ image=_convert_np_uint8(img_pred),
482
+ output_path=img_pred_file,
483
+ meta=meta_data,
484
+ )
485
+
486
+ # Save image rgb
487
+ if rgb_outputs:
488
+ rgb_file = os.path.join(
489
+ output_dir, "original_rgb_"
490
+ f"{os.path.splitext(os.path.basename(data_file))[0]}.tiff")
491
+ save_geotiff(
492
+ image=_convert_np_uint8(rgb_orig),
493
+ output_path=rgb_file,
494
+ meta=meta_data,
495
+ )
496
+
497
+
498
+if __name__ == "__main__":
499
+ parser = argparse.ArgumentParser("MAE run inference", add_help=False)
500
+
501
+ parser.add_argument(
502
+ "--data_file",
503
+ type=str,
504
+ default="./India_900498_S2Hand.tif",
505
+ help="Path to the file.",
506
+ )
507
+ parser.add_argument(
508
+ "--output_dir",
509
+ type=str,
510
+ default="output",
511
+ help="Path to the directory where to save outputs.",
512
+ )
513
+ parser.add_argument(
514
+ "--input_indices",
515
+ default=[1, 2, 3, 8, 11, 12],
516
+ type=int,
517
+ nargs="+",
518
+ help=
519
+ "0-based indices of the six Prithvi channels to be selected from the "
520
+ "input. By default selects [1,2,3,8,11,12] for S2L1C data.",
521
+ )
522
+ parser.add_argument(
523
+ "--rgb_outputs",
524
+ action="store_true",
525
+ help="If present, output files will only contain RGB channels. "
526
+ "Otherwise, all bands will be saved.",
527
+ )
528
+ args = parser.parse_args()
529
+
530
+ main(**vars(args))
tests/models/registry.py
+4
@@ -214,6 +214,10 @@ _EMBEDDING_EXAMPLE_MODELS = {
214
"Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
215
trust_remote_code=True),
216
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"), # noqa: E501
217
+ # The model on Huggingface is currently being updated,
218
+ # hence I temporarily mark it as not available online
219
+ "PrithviGeoSpatialMAE": _HfExamplesInfo("ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", # noqa: E501
220
+ is_available_online=False),
221
}
222
223
_CROSS_ENCODER_EXAMPLE_MODELS = {
vllm/attention/backends/placeholder_attn.py
+8
-3
@@ -320,9 +320,14 @@ class PlaceholderAttentionMetadataBuilder(
320
-1 if cuda graph is not used.
321
batch_size: The maybe padded batch size.
322
"""
323
- for inter_data in self.input_builder.inter_data_list:
324
- self._add_seq_group(inter_data,
325
- self.input_builder.chunked_prefill_enabled)
323
+
324
+ # Some input builders such as ModelInputForCPUBuilder do not have the
325
+ # "inter_data_list" attribute.
326
+ # Let's check inter_data_list exists before we reference it.
327
+ if hasattr(self.input_builder, "inter_data_list"):
328
+ for inter_data in self.input_builder.inter_data_list:
329
+ self._add_seq_group(inter_data,
330
+ self.input_builder.chunked_prefill_enabled)
331
332
device = self.runner.device
333
use_captured_graph = cuda_graph_pad_size != -1
vllm/inputs/preprocess.py
+17
-5
@@ -254,8 +254,14 @@ class InputPreprocessor:
254
Apply the model's multi-modal processor to a multi-modal prompt,
255
returning the corresponding token IDs and metadata.
256
"""
257
- tokenizer_group = self.get_tokenizer_group()
258
- tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
257
+ # At the moment on model (PrithviGeoSpatialMAE) requires to be
258
+ # initialized without a tokenizer while using also multi-modal
259
+ # input.
260
+ if not self.tokenizer:
261
+ tokenizer = None
262
+ else:
263
+ tokenizer_group = self.get_tokenizer_group()
264
+ tokenizer = tokenizer_group.get_lora_tokenizer(lora_request)
265
266
mm_processor = self.mm_registry.create_processor(
267
self.model_config, tokenizer)
@@ -273,9 +279,15 @@ class InputPreprocessor:
279
lora_request: Optional[LoRARequest],
280
) -> MultiModalInputs:
281
"""Async version of :meth:`_process_multimodal`."""
276
- tokenizer_group = self.get_tokenizer_group()
277
- tokenizer = await tokenizer_group.get_lora_tokenizer_async(lora_request
278
- )
282
+ # At the moment on model (PrithviGeoSpatialMAE) requires to be
283
+ # initialized without a tokenizer while using also multi-modal
284
+ # input.
285
+ if not self.tokenizer:
286
+ tokenizer = None
287
+ else:
288
+ tokenizer_group = self.get_tokenizer_group()
289
+ tokenizer = await tokenizer_group.get_lora_tokenizer_async(
290
+ lora_request)
291
292
mm_processor = self.mm_registry.create_processor(
293
self.model_config, tokenizer)
vllm/model_executor/models/prithvi_geospatial_mae.py
new
+238
@@ -0,0 +1,238 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# Copyright 2025 The vLLM team.
4
+# Copyright 2025 IBM.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Inference-only IBM/NASA Prithvi Geospatial model."""
18
+from typing import Iterable, List, Mapping, Optional, Set, Tuple, Union
19
+
20
+import torch
21
+import torch.nn as nn
22
+from transformers import BatchFeature
23
+
24
+from vllm.attention import AttentionMetadata
25
+from vllm.config import VllmConfig
26
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
+from vllm.model_executor.models.interfaces import (IsAttentionFree,
28
+ SupportsMultiModal)
29
+from vllm.model_executor.models.utils import AutoWeightsLoader
30
+from vllm.model_executor.pooling_metadata import PoolingMetadata
31
+from vllm.multimodal import MULTIMODAL_REGISTRY
32
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
33
+ MultiModalInputs, MultiModalKwargs)
34
+from vllm.multimodal.parse import MultiModalDataItems
35
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
36
+ BaseProcessingInfo, PromptReplacement)
37
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38
+from vllm.sequence import (IntermediateTensors, PoolerOutput,
39
+ PoolingSequenceGroupOutput)
40
+
41
+
42
+class PrithviGeoSpatialMAEProcessingInfo(BaseProcessingInfo):
43
+
44
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
45
+ return {"image": None}
46
+
47
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
48
+ pass
49
+
50
+
51
+class PrithviGeoSpatialMAEInputBuilder(
52
+ BaseDummyInputsBuilder[PrithviGeoSpatialMAEProcessingInfo]):
53
+
54
+ def get_dummy_processor_inputs(
55
+ self,
56
+ seq_len: int,
57
+ mm_counts: Mapping[str, int],
58
+ ) -> ProcessorInputs:
59
+ return ProcessorInputs(
60
+ prompt_text="",
61
+ # This model input is fixed and is in the form of a torch Tensor.
62
+ # The size of pixel_values might change in the cases where we resize
63
+ # the input but never exceeds the dimensions below.
64
+ mm_data={
65
+ "pixel_values": torch.full((1, 6, 512, 512), 1.0),
66
+ "location_coords": torch.full((1, 2), 1.0)
67
+ })
68
+
69
+
70
+class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
71
+
72
+ def _get_mm_fields_config(
73
+ self,
74
+ hf_inputs: BatchFeature,
75
+ hf_processor_mm_kwargs: Mapping[str, object],
76
+ ) -> Mapping[str, MultiModalFieldConfig]:
77
+ return dict(
78
+ pixel_values=MultiModalFieldConfig.batched("image"),
79
+ location_coords=MultiModalFieldConfig.batched("image"),
80
+ )
81
+
82
+ def _get_prompt_replacements(
83
+ self,
84
+ mm_items: MultiModalDataItems,
85
+ hf_processor_mm_kwargs: Mapping[str, object],
86
+ out_mm_kwargs: MultiModalKwargs,
87
+ ) -> list[PromptReplacement]:
88
+ pass
89
+
90
+ def _get_mm_fields_config(
91
+ self,
92
+ hf_inputs: BatchFeature,
93
+ hf_processor_mm_kwargs: Mapping[str, object],
94
+ ) -> Mapping[str, MultiModalFieldConfig]:
95
+ pass
96
+
97
+ def apply(
98
+ self,
99
+ prompt: Union[str, list[int]],
100
+ mm_data: MultiModalDataDict,
101
+ hf_processor_mm_kwargs: Mapping[str, object],
102
+ ) -> MultiModalInputs:
103
+ mm_kwargs = {}
104
+
105
+ for k, v in mm_data.items():
106
+ mm_kwargs[k] = v
107
+
108
+ return MultiModalInputs(
109
+ type="multimodal",
110
+ prompt=prompt,
111
+ prompt_token_ids=[1],
112
+ mm_kwargs=MultiModalKwargs(mm_kwargs),
113
+ mm_placeholders={},
114
+ )
115
+
116
+
117
+@MULTIMODAL_REGISTRY.register_processor(
118
+ PrithviGeoSpatialMAEMultiModalProcessor,
119
+ info=PrithviGeoSpatialMAEProcessingInfo,
120
+ dummy_inputs=PrithviGeoSpatialMAEInputBuilder)
121
+class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal):
122
+ """ Prithvi Masked Autoencoder"""
123
+
124
+ def _instantiate_model(self, config: dict) -> nn.Module | None:
125
+
126
+ # We might be able/need to support different tasks with this same model
127
+ if config["task_args"]["task"] == "SemanticSegmentationTask":
128
+ from terratorch.cli_tools import SemanticSegmentationTask
129
+ task = SemanticSegmentationTask(
130
+ config["model_args"],
131
+ config["task_args"]["model_factory"],
132
+ loss=config["task_args"]["loss"],
133
+ lr=config["task_args"]["lr"],
134
+ ignore_index=config["task_args"]["ignore_index"],
135
+ optimizer=config["task_args"]["optimizer"],
136
+ optimizer_hparams=config["optimizer_params"],
137
+ scheduler=config["task_args"]["scheduler"],
138
+ scheduler_hparams=config["scheduler_params"],
139
+ plot_on_val=config["task_args"]["plot_on_val"],
140
+ freeze_decoder=config["task_args"]["freeze_decoder"],
141
+ freeze_backbone=config["task_args"]["freeze_backbone"])
142
+
143
+ return task.model
144
+ else:
145
+ return None
146
+
147
+ def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
148
+ super().__init__()
149
+
150
+ # the actual model is dynamically instantiated using terratorch
151
+ # allowing us to perform changes to the model architecture
152
+ # at startup time (e.g., change the model decoder class.)
153
+ self.model = self._instantiate_model(
154
+ vllm_config.model_config.hf_config.to_dict()["pretrained_cfg"])
155
+ if self.model is None:
156
+ raise ValueError(
157
+ "Unsupported task."
158
+ "Only SemanticSegmentationTask is supported for now"
159
+ "by PrithviGeospatialMAE.")
160
+
161
+ def _parse_and_validate_multimodal_data(
162
+ self, **kwargs) -> Tuple[torch.Tensor, torch.Tensor | None]:
163
+
164
+ pixel_values = kwargs.pop("pixel_values", None)
165
+ if not isinstance(pixel_values, torch.Tensor):
166
+ raise ValueError(f"Incorrect type of pixel_values. "
167
+ f"Got type: {type(pixel_values)}")
168
+ pixel_values = torch.unbind(pixel_values, dim=0)[0]
169
+
170
+ location_coords = kwargs.pop("location_coords", None)
171
+ if not isinstance(location_coords, torch.Tensor):
172
+ raise ValueError(f"Incorrect type of location_coords. "
173
+ f"Got type: {type(location_coords)}")
174
+ location_coords = torch.unbind(location_coords, dim=0)[0]
175
+ if location_coords.shape == torch.Size([0]):
176
+ location_coords = None
177
+
178
+ return pixel_values, location_coords
179
+
180
+ def forward(
181
+ self,
182
+ input_ids: Optional[torch.Tensor],
183
+ positions: torch.Tensor,
184
+ kv_caches: List[torch.Tensor],
185
+ attn_metadata: AttentionMetadata,
186
+ intermediate_tensors: Optional[IntermediateTensors] = None,
187
+ inputs_embeds: Optional[torch.Tensor] = None,
188
+ **kwargs: object,
189
+ ):
190
+
191
+ pixel_values, location_coords = (
192
+ self._parse_and_validate_multimodal_data(**kwargs))
193
+ model_output = self.model(pixel_values,
194
+ location_coords=location_coords)
195
+
196
+ return model_output.output
197
+
198
+ def pooler(
199
+ self,
200
+ hidden_states: torch.Tensor,
201
+ pooling_metadata: PoolingMetadata,
202
+ ) -> Optional[PoolerOutput]:
203
+ return PoolerOutput([PoolingSequenceGroupOutput(hidden_states)])
204
+
205
+ def load_weights(self, weights: Iterable[Tuple[str,
206
+ torch.Tensor]]) -> Set[str]:
207
+ params_list = []
208
+ model_buffers = dict(self.named_buffers())
209
+ loaded_buffers = []
210
+ for key, value in weights:
211
+ if key == "state_dict":
212
+ weights_to_parse = value
213
+ for name, weight in weights_to_parse.items():
214
+ if "pos_embed" in name:
215
+ continue
216
+
217
+ if "_timm_module." in name:
218
+ name = name.replace("_timm_module.", "")
219
+
220
+ # this model requires a couple of buffers to be loaded
221
+ # that are not loadable with the AutoWeightsLoader
222
+ if name in model_buffers:
223
+ if "_timm_module." in name:
224
+ name = name.replace("_timm_module.", "")
225
+ buffer = model_buffers[name]
226
+ weight_loader = getattr(buffer, "weight_loader",
227
+ default_weight_loader)
228
+ weight_loader(buffer, weight)
229
+ loaded_buffers.append(name)
230
+ else:
231
+ params_list.append((name, weight))
232
+ break
233
+
234
+ # Load the remaining model parameters
235
+ loader = AutoWeightsLoader(self)
236
+ autoloaded_weights = loader.load_weights(params_list)
237
+
238
+ return autoloaded_weights.union(set(loaded_buffers))
vllm/model_executor/models/registry.py
+4
@@ -137,6 +137,10 @@ _EMBEDDING_MODELS = {
137
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
138
# [Auto-converted (see adapters.py)]
139
"Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),
140
+ # Technically PrithviGeoSpatialMAE is a model that works on images, both in
141
+ # input and output. I am adding it here because it piggy-backs on embedding
142
+ # models for the time being.
143
+ "PrithviGeoSpatialMAE": ("prithvi_geospatial_mae", "PrithviGeoSpatialMAE"),
144
}
145
146
_CROSS_ENCODER_MODELS = {
vllm/worker/pooling_model_runner.py
+10
-1
@@ -74,7 +74,16 @@ class PoolingModelRunner(
74
prefill_meta = model_input.attn_metadata.prefill_metadata
75
decode_meta = model_input.attn_metadata.decode_metadata
76
virtual_engine = model_input.virtual_engine
77
- if prefill_meta is None and decode_meta.use_cuda_graph:
77
+ # Pooling models are (ab-)used also to integrate non text models that
78
+ # are not autoregressive (PrithviGeosaptialMAE).
79
+ # These model might not use attention and do not really have a prefill
80
+ # and decode phase. The model input is processed in one shot and both
81
+ # decode_metadata and prefill_metadata would be None for such models.
82
+ # See the PlaceholderAttentionMetadata class.
83
+ # TODO: Figure out if cuda_graph is of any use for these models and
84
+ # explore how to leverage it.
85
+ if (prefill_meta is None and decode_meta is not None
86
+ and decode_meta.use_cuda_graph):
87
assert model_input.input_tokens is not None
88
graph_batch_size = model_input.input_tokens.shape[0]
89
model_executable = self.graph_runners[virtual_engine][