1
+###############################################################################
2
+# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company
3
+###############################################################################
4
+
5
+import collections
6
+import contextlib
7
+import dataclasses
8
+import functools
9
+import gc
10
+import itertools
11
+import math
12
+import operator
13
+import os
14
+import time
15
+from array import array
16
+from dataclasses import dataclass, field
17
+from enum import IntEnum
18
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple,
19
+ Optional, Set, Tuple, Type, TypeVar, Union)
20
+
21
+import habana_frameworks.torch as htorch
22
+import habana_frameworks.torch.internal.bridge_config as bc
23
+import torch
24
+from vllm_hpu_extension.ops import LoraMask as LoraMask
25
+from vllm_hpu_extension.profiler import (HabanaHighLevelProfiler,
26
+ HabanaMemoryProfiler, format_bytes)
27
+
28
+from vllm.attention import AttentionMetadata, get_attn_backend
29
+from vllm.config import DeviceConfig, VllmConfig
30
+from vllm.distributed.parallel_state import get_world_group
31
+from vllm.logger import init_logger
32
+from vllm.lora.layers import LoRAMapping
33
+from vllm.lora.request import LoRARequest
34
+from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
35
+from vllm.model_executor import SamplingMetadata
36
+from vllm.model_executor.layers.sampler import SamplerOutput
37
+from vllm.model_executor.model_loader import get_model
38
+from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
39
+ MultiModalInputs)
40
+from vllm.sampling_params import SamplingParams
41
+from vllm.sequence import (IntermediateTensors, SequenceData,
42
+ SequenceGroupMetadata)
43
+from vllm.utils import is_pin_memory_available, make_tensor_with_pad
44
+from vllm.worker.model_runner_base import (
45
+ ModelRunnerBase, ModelRunnerInputBase,
46
+ _add_attn_metadata_broadcastable_dict,
47
+ _add_sampling_metadata_broadcastable_dict,
48
+ _init_attn_metadata_from_tensor_dict,
49
+ _init_sampling_metadata_from_tensor_dict)
50
+
51
+if TYPE_CHECKING:
52
+ from vllm.attention.backends.abstract import AttentionBackend
53
+
54
+logger = init_logger(__name__)
55
+
56
+_TYPE_CACHE = {}
57
+# These values are assumed to be zero in several places.
58
+# Use caution when updating them!
59
+_PAD_SLOT_ID = 0
60
+_PAD_BLOCK_ID = 0
61
+
62
+LORA_WARMUP_RANK = 8
63
+
64
+
65
+class Singleton(type):
66
+ _instances: Dict[type, object] = {}
67
+
68
+ def __call__(cls, *args, **kwargs):
69
+ if cls not in cls._instances:
70
+ cls._instances[cls] = super(Singleton,
71
+ cls).__call__(*args, **kwargs)
72
+ return cls._instances[cls]
73
+
74
+
75
+@dataclass
76
+class HPUBucketingGlobalState(metaclass=Singleton):
77
+ prompt_bs_bucket_cfg: Tuple[int, int, int] = field(init=False)
78
+ decode_bs_bucket_cfg: Tuple[int, int, int] = field(init=False)
79
+ prompt_seq_bucket_cfg: Tuple[int, int, int] = field(init=False)
80
+ decode_block_bucket_cfg: Tuple[int, int, int] = field(init=False)
81
+ prompt_buckets: List[Tuple[int, int]] = field(init=False)
82
+ decode_buckets: List[Tuple[int, int]] = field(init=False)
83
+
84
+
85
+def subtuple(obj: object,
86
+ typename: str,
87
+ to_copy: List[str],
88
+ to_override: Optional[Dict[str, object]] = None):
89
+ if obj is None:
90
+ return None
91
+ if to_override is None:
92
+ to_override = {}
93
+ fields = set(to_copy) | set(to_override.keys())
94
+ values = {f: to_override.get(f, getattr(obj, f)) for f in fields}
95
+ if typename not in _TYPE_CACHE:
96
+ _TYPE_CACHE[typename] = collections.namedtuple(typename,
97
+ ' '.join(fields))
98
+ return _TYPE_CACHE[typename](**values)
99
+
100
+
101
+def read_bucket_settings(phase: str, dim: str, **defaults):
102
+ """Read bucketing configuration from env variables.
103
+
104
+ phase is either 'prompt' or 'decode'
105
+ dim is either 'bs', 'seq' or 'block'
106
+ param is either 'min', 'step' or 'max'
107
+ example env variable: VLLM_DECODE_BS_BUCKET_STEP=128
108
+ """
109
+ params = ['min', 'step', 'max']
110
+ env_vars = [f'VLLM_{phase}_{dim}_BUCKET_{p}'.upper() for p in params]
111
+ default_values = [defaults[p] for p in params]
112
+ values = [
113
+ int(os.environ.get(e, d)) for e, d in zip(env_vars, default_values)
114
+ ]
115
+ for e, v, d in zip(env_vars, values, default_values):
116
+ logger.info('%s=%s (default:%s)', e, v, d)
117
+ return values
118
+
119
+
120
+def warmup_range(config: Tuple[int, int, int]):
121
+ """Generate a warmup range.
122
+
123
+ Start from bmin and multiply by 2 until you reach bstep.
124
+ Then, increase the values in the range by the value of bstep until you
125
+ reach bmax.
126
+
127
+ Example:
128
+ bmin = 2, bstep = 32, bmax = 64
129
+ => ramp_up = (2, 4, 8, 16)
130
+ => stable = (32, 64)
131
+ => return ramp_up + stable => (2, 4, 8, 16, 32, 64)
132
+ """
133
+ bmin, bstep, bmax = config
134
+ assert bmin <= bmax, ("Min. batch size cannot be greater than max. "
135
+ "batch size. If you want to skip warmup, "
136
+ "set VLLM_SKIP_WARMUP=true")
137
+ base = itertools.repeat(2)
138
+ ramp_up_acc = itertools.accumulate(base, func=operator.mul, initial=bmin)
139
+ ramp_up_tw = itertools.takewhile(lambda x: x < bstep and x <= bmax, \
140
+ ramp_up_acc)
141
+ stable = range(bstep, bmax + 1, bstep)
142
+ buckets = list(ramp_up_tw) + list(stable)
143
+ return list(filter(lambda bucket: bucket >= bmin, buckets))
144
+
145
+
146
+def generate_prompt_buckets(bs_bucket_config,
147
+ seq_bucket_config,
148
+ max_num_batched_tokens=None):
149
+ buckets = list(
150
+ itertools.product(warmup_range(bs_bucket_config),
151
+ warmup_range(seq_bucket_config)))
152
+ if len(buckets) == 0:
153
+ msg = ("No buckets could be captured with following config "
154
+ f"(min, step, max_warmup): "
155
+ f"bs:{bs_bucket_config}, "
156
+ f"seq:{seq_bucket_config}")
157
+ raise ValueError(msg)
158
+
159
+ filtered_buckets = buckets
160
+ if max_num_batched_tokens is not None:
161
+ # Remove buckets exceeding batch token budget
162
+ filtered_buckets = list(
163
+ filter(
164
+ lambda bucket: bucket[0] * bucket[1] <= max_num_batched_tokens,
165
+ buckets))
166
+
167
+ if len(filtered_buckets) == 0:
168
+ # we can handle this if we ignore max_num_batched_tokens
169
+ min_bucket_bs, min_bucket_seq = min(buckets,
170
+ key=lambda b: (b[0] * b[1]))
171
+ min_reqd_budget = min_bucket_bs * min_bucket_seq
172
+ msg = (
173
+ "The current bucketing configuration "
174
+ f"(min, step, max_warmup): "
175
+ f"bs:{bs_bucket_config}, "
176
+ f"seq:{seq_bucket_config} cannot be used with specified "
177
+ f"max_num_batched_tokens ({max_num_batched_tokens}), as the "
178
+ f"smallest bucket ({min_reqd_budget}) would exceed token "
179
+ "budget. Please increase max_num_batched_tokens or decrease "
180
+ "bucket minimum Ignoring max_num_batched_tokens at risk of "
181
+ "out-of-memory errors.")
182
+ logger.error(msg)
183
+ return list(
184
+ sorted(buckets, key=lambda b: (b[0] * b[1], b[1], b[0]))), []
185
+
186
+ captured_buckets = list(
187
+ sorted(filtered_buckets, key=lambda b: (b[0] * b[1], b[1], b[0])))
188
+ omitted_buckets = list(
189
+ sorted([x for x in buckets if x not in filtered_buckets]))
190
+ return captured_buckets, omitted_buckets
191
+
192
+
193
+def generate_decode_buckets(bs_bucket_config, blocks_bucket_config,
194
+ max_blocks):
195
+ buckets = []
196
+ bs_buckets = warmup_range(bs_bucket_config)
197
+ block_buckets = warmup_range(blocks_bucket_config)
198
+ bmin, bstep, bmax = blocks_bucket_config
199
+ last_bucket = round_up(max_blocks, bstep)
200
+ for bs in bs_buckets:
201
+ for blocks in block_buckets:
202
+ if blocks < bs:
203
+ continue
204
+ if blocks > last_bucket:
205
+ break
206
+ buckets.append((bs, blocks))
207
+ return list(sorted(buckets, key=lambda b: (b[0] * b[1], b[1], b[0])))
208
+
209
+
210
+def next_pow2(value: int, base: int):
211
+ res = base
212
+ while value > 1:
213
+ value = (value + 1) // 2
214
+ res *= 2
215
+ return res
216
+
217
+
218
+def round_up(value: int, k: int):
219
+ return (value + k - 1) // k * k
220
+
221
+
222
+def find_bucket(value: int, config: Tuple[int, int, int]):
223
+ bmin, bstep, _ = config
224
+ next_step = round_up(value, bstep)
225
+ next_pow = next_pow2(value, bmin)
226
+ return max(bmin, min(next_step, next_pow))
227
+
228
+
229
+def align_workers(value, op):
230
+ group = get_world_group().cpu_group
231
+ world_size = torch.distributed.get_world_size()
232
+ if world_size <= 1:
233
+ return value
234
+ value_t = torch.tensor(value, device='cpu')
235
+ torch.distributed.all_reduce(value_t, op=op, group=group)
236
+ return value_t.item()
237
+
238
+
239
+def setup_profiler():
240
+ schedule = torch.profiler.schedule(wait=0, warmup=2, active=1, repeat=1)
241
+ DEVICE = 'hpu'
242
+ activities = [torch.profiler.ProfilerActivity.CPU]
243
+ activities.extend([torch.profiler.ProfilerActivity.HPU] if DEVICE ==
244
+ 'hpu' else [])
245
+ #from habana_frameworks.torch.activity_profiler import DebugActivity
246
+ #debug_activities=[DebugActivity.BRIDGE_FUNCTION_CALLS]
247
+
248
+ profiler = torch.profiler.profile(
249
+ schedule=schedule,
250
+ activities=activities,
251
+ #debug_activities=debug_activities,
252
+ on_trace_ready=torch.profiler.tensorboard_trace_handler('.',
253
+ use_gzip=True),
254
+ record_shapes=False,
255
+ with_stack=True)
256
+ return profiler
257
+
258
+
259
+def pad_list(list, k, v):
260
+ target_len = round_up(len(list), k)
261
+ padding = target_len - len(list)
262
+ return list + [v] * padding
263
+
264
+
265
+def precompute_indices_and_offsets(block_size, slot_mapping, is_prompt):
266
+ slot_mapping = slot_mapping.flatten()
267
+ indices = torch.div(slot_mapping, block_size, rounding_mode="floor")
268
+ if is_prompt:
269
+ indices = indices.unflatten(0, (-1, block_size))[:, 0]
270
+ offsets = None
271
+ else:
272
+ offsets = torch.fmod(slot_mapping, block_size)
273
+ return indices, offsets
274
+
275
+
276
+class HpuModelAdapter():
277
+
278
+ def __init__(self, model, block_size, dtype, enforce_eager):
279
+ self.model = model
280
+ self.prefill_use_fusedsdpa = os.getenv('VLLM_PROMPT_USE_FUSEDSDPA',
281
+ '0').lower() in ['1', 'true']
282
+ self.block_size = block_size
283
+ self.dtype = dtype
284
+ if not htorch.utils.internal.is_lazy() and not enforce_eager:
285
+ self.model = torch.compile(self.model,
286
+ backend='hpu_backend',
287
+ dynamic=False)
288
+
289
+ def _set_attn_bias(self, attn_metadata, batch_size, seq_len, device,
290
+ dtype):
291
+ prefill_metadata = attn_metadata
292
+ if prefill_metadata is None or self.prefill_use_fusedsdpa:
293
+ return attn_metadata
294
+
295
+ seq_lens_t = prefill_metadata.seq_lens_tensor
296
+ len_mask = (torch.arange(0, seq_len, device=device,
297
+ dtype=torch.int32).view(1, seq_len).ge(
298
+ seq_lens_t.unsqueeze(-1)).view(
299
+ batch_size, 1, 1, seq_len))
300
+ causal_mask = torch.triu(torch.ones((batch_size, 1, seq_len, seq_len),
301
+ device=device,
302
+ dtype=torch.bool),
303
+ diagonal=1)
304
+ mask = causal_mask.logical_or(len_mask)
305
+ attn_bias = (torch.zeros_like(mask, dtype=dtype).masked_fill_(
306
+ mask, -math.inf))
307
+ attn_metadata = prefill_metadata._replace(attn_bias=attn_bias)
308
+ return attn_metadata
309
+
310
+ def _set_block_mapping(self, metadata, batch_size, device, dtype):
311
+ mask = torch.arange(0,
312
+ self.block_size,
313
+ device=device,
314
+ dtype=torch.int32).unsqueeze(0)
315
+ mask = mask >= metadata.block_usage.unsqueeze(-1)
316
+ attn_bias = (torch.zeros_like(mask, dtype=dtype).masked_fill_(
317
+ mask, -math.inf))
318
+ block_mapping = torch.nn.functional.one_hot(metadata.block_mapping,
319
+ num_classes=batch_size)
320
+ block_mapping = block_mapping.to(dtype)
321
+ metadata = metadata._replace(block_mapping=block_mapping,
322
+ attn_bias=attn_bias)
323
+ return metadata
324
+
325
+ def _update_metadata(self, attn_metadata, batch_size, seq_len, device,
326
+ dtype):
327
+ if attn_metadata.is_prompt:
328
+ meta = attn_metadata
329
+ attn_metadata = self._set_attn_bias(meta, batch_size, seq_len,
330
+ device, dtype)
331
+ else:
332
+ meta = attn_metadata
333
+ attn_metadata = self._set_block_mapping(meta, batch_size, device,
334
+ dtype)
335
+ return attn_metadata
336
+
337
+ def forward(self, *args, **kwargs):
338
+ kwargs = kwargs.copy()
339
+ selected_token_indices = kwargs.pop('selected_token_indices')
340
+ if 'warmup_mode' in kwargs:
341
+ kwargs.pop('warmup_mode')
342
+ input_ids = kwargs['input_ids']
343
+ kwargs['attn_metadata'] = self._update_metadata(
344
+ kwargs['attn_metadata'], input_ids.size(0), input_ids.size(1),
345
+ input_ids.device, self.dtype)
346
+ LoraMask.setLoraMask(kwargs.pop('lora_mask'))
347
+ hidden_states = self.model(*args, **kwargs)
348
+ hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
349
+ hidden_states = hidden_states.index_select(0, selected_token_indices)
350
+ return hidden_states
351
+
352
+ def compute_logits(self, *args, **kwargs):
353
+ return self.model.compute_logits(*args, **kwargs)
354
+
355
+ def sample(self, *args, **kwargs):
356
+ return self.model.sample(*args, **kwargs)
357
+
358
+
359
+class PreparePromptMetadata(NamedTuple):
360
+ input_tokens: torch.Tensor
361
+ input_positions: List[List[int]]
362
+ attn_metadata: Optional[AttentionMetadata]
363
+ seq_lens: List[int]
364
+ query_lens: List[int]
365
+ lora_index_mapping: List[List[int]]
366
+ lora_prompt_mapping: List[List[int]]
367
+ lora_requests: Set[LoRARequest]
368
+ multi_modal_kwargs: Optional[Dict[str, BatchedTensorInputs]]
369
+ slot_mapping: List[List[int]]
370
+ lora_ids: List[int]
371
+
372
+ @classmethod
373
+ def empty(cls):
374
+ return PreparePromptMetadata(input_tokens=[],
375
+ input_positions=[],
376
+ attn_metadata=None,
377
+ seq_lens=[],
378
+ query_lens=[],
379
+ lora_index_mapping=[],
380
+ lora_prompt_mapping=[],
381
+ lora_requests=set(),
382
+ multi_modal_kwargs=None,
383
+ slot_mapping=[],
384
+ lora_ids=[])
385
+
386
+
387
+class PrepareDecodeMetadata(NamedTuple):
388
+ input_tokens: torch.Tensor
389
+ input_positions: List[List[int]]
390
+ attn_metadata: Optional[AttentionMetadata]
391
+ lora_index_mapping: List[List[int]]
392
+ lora_prompt_mapping: List[List[int]]
393
+ lora_requests: Set[LoRARequest]
394
+ slot_mapping: List[List[int]]
395
+ lora_ids: List[int]
396
+
397
+ @classmethod
398
+ def empty(cls):
399
+ return PrepareDecodeMetadata(input_tokens=[],
400
+ input_positions=[],
401
+ attn_metadata=None,
402
+ lora_index_mapping=[],
403
+ lora_prompt_mapping=[],
404
+ lora_requests=set(),
405
+ slot_mapping=[],
406
+ lora_ids=[])
407
+
408
+
409
+# How batches are constructed.
410
+class BatchType(IntEnum):
411
+ # Every batch is prefill.
412
+ PREFILL = 0
413
+ # Every batch is decode.
414
+ DECODE = 1
415
+ # Batch is a mixture of prefill and decode.
416
+ MIXED = 2
417
+
418
+
419
+TModelInputForHPU = TypeVar('TModelInputForHPU', bound="ModelInputForHPU")
420
+
421
+
422
+@dataclasses.dataclass(frozen=True)
423
+class ModelInputForHPU(ModelRunnerInputBase):
424
+ """
425
+ This base class contains metadata needed for the base model forward pass
426
+ but not metadata for possible additional steps, e.g., sampling. Model
427
+ runners that run additional steps should subclass this method to add
428
+ additional fields.
429
+ """
430
+ input_tokens: Optional[torch.Tensor] = None
431
+ input_positions: Optional[torch.Tensor] = None
432
+ seq_lens: Optional[List[int]] = None
433
+ query_lens: Optional[List[int]] = None
434
+ lora_mapping: Optional["LoRAMapping"] = None
435
+ lora_requests: Optional[Set[LoRARequest]] = None
436
+ attn_metadata: Optional["AttentionMetadata"] = None
437
+ multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
438
+ real_batch_size: Optional[int] = None
439
+ batch_size_padded: Optional[int] = None
440
+ virtual_engine: int = 0
441
+ lora_ids: Optional[List[int]] = None
442
+ async_callback: Optional[Callable] = None
443
+
444
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
445
+ tensor_dict = {
446
+ "input_tokens": self.input_tokens,
447
+ "input_positions": self.input_positions,
448
+ "lora_requests": self.lora_requests,
449
+ "lora_mapping": self.lora_mapping,
450
+ "multi_modal_kwargs": self.multi_modal_kwargs,
451
+ "real_batch_size": self.real_batch_size,
452
+ "batch_size_padded": self.batch_size_padded,
453
+ "virtual_engine": self.virtual_engine,
454
+ "lora_ids": self.lora_ids,
455
+ }
456
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
457
+ return tensor_dict
458
+
459
+ @classmethod
460
+ def from_broadcasted_tensor_dict(
461
+ cls: Type[TModelInputForHPU],
462
+ tensor_dict: Dict[str, Any],
463
+ attn_backend: Optional["AttentionBackend"] = None,
464
+ ) -> TModelInputForHPU:
465
+ if attn_backend is not None:
466
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
467
+ attn_backend, tensor_dict)
468
+ return cls(**tensor_dict)
469
+
470
+
471
+@dataclasses.dataclass(frozen=True)
472
+class ModelInputForHPUWithSamplingMetadata(ModelInputForHPU):
473
+ """
474
+ Used by the ModelRunner.
475
+ """
476
+ sampling_metadata: Optional["SamplingMetadata"] = None
477
+ # Used for speculative decoding. We do not broadcast it because it is only
478
+ # used by the driver worker.
479
+ is_prompt: Optional[bool] = None
480
+
481
+ def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
482
+ tensor_dict = {
483
+ "input_tokens": self.input_tokens,
484
+ "input_positions": self.input_positions,
485
+ "lora_requests": self.lora_requests,
486
+ "lora_mapping": self.lora_mapping,
487
+ "multi_modal_kwargs": self.multi_modal_kwargs,
488
+ "lora_ids": self.lora_ids,
489
+ }
490
+ _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
491
+ _add_sampling_metadata_broadcastable_dict(tensor_dict,
492
+ self.sampling_metadata)
493
+ return tensor_dict
494
+
495
+ @classmethod
496
+ def from_broadcasted_tensor_dict(
497
+ cls,
498
+ tensor_dict: Dict[str, Any],
499
+ attn_backend: Optional["AttentionBackend"] = None,
500
+ ) -> "ModelInputForHPUWithSamplingMetadata":
501
+ tensor_dict = _init_sampling_metadata_from_tensor_dict(tensor_dict)
502
+ # FIXME(kzawora): this fails for whatever reason - why?
503
+ if attn_backend is not None:
504
+ tensor_dict = _init_attn_metadata_from_tensor_dict(
505
+ attn_backend, tensor_dict)
506
+ return cls(**tensor_dict)
507
+
508
+
509
+class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
510
+ """
511
+ Helper class for shared methods between GPU model runners.
512
+ """
513
+ _model_input_cls: Type[TModelInputForHPU]
514
+
515
+ def __init__(
516
+ self,
517
+ vllm_config: VllmConfig,
518
+ is_driver_worker: bool = False,
519
+ return_hidden_states: bool = False,
520
+ ):
521
+ ModelRunnerBase.__init__(self, vllm_config=vllm_config)
522
+ self.is_driver_worker = is_driver_worker
523
+ self.return_hidden_states = return_hidden_states
524
+
525
+ self.sliding_window = (self.model_config.get_sliding_window()
526
+ if self.model_config is not None else None)
527
+ self.device_config = (self.device_config if self.device_config
528
+ is not None else DeviceConfig())
529
+ self.device = self.device_config.device
530
+ self.enforce_eager = self.model_config.enforce_eager
531
+ self.max_num_seqs = self.scheduler_config.max_num_seqs
532
+ # NOTE(kzawora): Change that to scheduler_config.max_num_prefill_seqs
533
+ # once padding-aware scheduling gets merged
534
+ self.max_num_prefill_seqs = 64
535
+ self.max_model_len = self.scheduler_config.max_model_len
536
+ self.max_num_batched_tokens = \
537
+ self.scheduler_config.max_num_batched_tokens
538
+ self.block_size = self.cache_config.block_size
539
+
540
+ self.pin_memory = is_pin_memory_available()
541
+ self.kv_cache_dtype = self.cache_config.cache_dtype
542
+
543
+ self.attn_backend = get_attn_backend(
544
+ self.model_config.get_head_size(),
545
+ self.model_config.dtype,
546
+ self.kv_cache_dtype,
547
+ self.block_size,
548
+ self.model_config.is_attention_free,
549
+ )
550
+
551
+ # Lazy initialization
552
+ self.lora_manager: LRUCacheWorkerLoRAManager = None
553
+ self.model: torch.nn.Module = None
554
+ self.inc_initialized_successfully = False
555
+
556
+ # Profiler stats
557
+ self.profiler = HabanaHighLevelProfiler()
558
+ self.profiler_counter_helper = HabanaProfilerCounterHelper()
559
+ self.seen_configs: set = set()
560
+ self._mem_margin: Optional[int] = None
561
+ self.bucketing_global_state = HPUBucketingGlobalState()
562
+ self._setup_buckets()
563
+ self._set_gc_threshold()
564
+
565
+ def _set_gc_threshold(self) -> None:
566
+ # Read https://docs.python.org/3/library/gc.html#gc.set_threshold
567
+ # for comprehensive description of gc generations.
568
+ # We can either use VLLM_GC_THR_GEN[0-2] (this has higher priority)
569
+ # to set particular generation threshold or use simpler
570
+ # VLLM_GC_THR_MULTIPLIER to multiply default values.
571
+ default_gc_thrs = list(gc.get_threshold())
572
+ requested_gc_thrs = [0] * len(default_gc_thrs)
573
+ for i in range(len(default_gc_thrs)):
574
+ requested_gc_thrs[i] = int(
575
+ os.environ.get(f'VLLM_GC_THR_GEN{i}', default_gc_thrs[i]))
576
+ if requested_gc_thrs == default_gc_thrs:
577
+ gc_thr_multiplier = int(os.environ.get('VLLM_GC_THR_MULTIPLIER',
578
+ 2))
579
+ requested_gc_thrs = [
580
+ t * gc_thr_multiplier for t in default_gc_thrs
581
+ ]
582
+ gc.set_threshold(*requested_gc_thrs)
583
+
584
+ # Multi-modal data support
585
+ self.multi_modal_input_mapper = MULTIMODAL_REGISTRY \
586
+ .create_input_mapper(self.model_config)
587
+
588
+ self.skip_warmup = os.environ.get('VLLM_SKIP_WARMUP',
589
+ 'false').lower() == 'true'
590
+
591
+ def load_model(self) -> None:
592
+ import habana_frameworks.torch.core as htcore
593
+ if self.model_config.quantization == 'inc' or \
594
+ self.model_config.quantization == 'fp8':
595
+ htcore.hpu_set_env()
596
+ with HabanaMemoryProfiler() as m:
597
+ with HabanaMemoryProfiler() as m_getmodel:
598
+ self.model = get_model(vllm_config=self.vllm_config)
599
+ msg = ("Pre-loading model weights on "
600
+ f"{next(self.model.parameters()).device} "
601
+ f"took {m_getmodel.get_summary_string()}")
602
+ logger.info(msg)
603
+
604
+ if self.lora_config:
605
+ assert hasattr(self.model, "supported_lora_modules"
606
+ ) and self.model.supported_lora_modules, (
607
+ "Model does not support LoRA")
608
+ assert hasattr(self.model, "embedding_modules"
609
+ ), "Model does not have embedding_modules"
610
+ assert hasattr(
611
+ self.model, "embedding_padding_modules"
612
+ ), "Model does not have embedding_padding_modules"
613
+ self.lora_manager = LRUCacheWorkerLoRAManager(
614
+ self.scheduler_config.max_num_seqs,
615
+ self.scheduler_config.max_num_batched_tokens,
616
+ self.vocab_size, self.lora_config, self.device,
617
+ self.model.embedding_modules,
618
+ self.model.embedding_padding_modules)
619
+ self.model = self.lora_manager.create_lora_manager(self.model)
620
+
621
+ if self.model_config.quantization == 'inc':
622
+ logger.info("Preparing model with INC..")
623
+ with HabanaMemoryProfiler() as m_inc:
624
+ from neural_compressor.torch.quantization import (
625
+ FP8Config, convert, prepare)
626
+ config = FP8Config.from_json_file(
627
+ os.getenv("QUANT_CONFIG", ""))
628
+ if config.measure:
629
+ self.model = prepare(self.model, config)
630
+ elif config.quantize:
631
+ self.model = convert(self.model, config)
632
+ htcore.hpu_initialize(self.model,
633
+ mark_only_scales_as_const=True)
634
+ self.inc_initialized_successfully = True
635
+ logger.info("Preparing model with INC took %s",
636
+ m_inc.get_summary_string())
637
+ else:
638
+ self.model = self.model.to("hpu")
639
+ htcore.mark_step()
640
+ torch.hpu.synchronize()
641
+
642
+ with HabanaMemoryProfiler() as m_wrap:
643
+ self.model = _maybe_wrap_in_hpu_graph(
644
+ self.model,
645
+ self.block_size,
646
+ dtype=self.model_config.dtype,
647
+ enforce_eager=self.enforce_eager)
648
+ msg = f"Wrapping in HPU Graph took {m_wrap.get_summary_string()}"
649
+ logger.info(msg)
650
+
651
+ self.model_memory_usage = m.consumed_device_memory
652
+ msg = f"Loading model weights took in total {m.get_summary_string()}"
653
+ logger.info(msg)
654
+
655
+ def _use_graphs(self, batch_size, seq_len, is_prompt):
656
+ if self.enforce_eager:
657
+ return False
658
+ if self.skip_warmup:
659
+ return True
660
+ return (batch_size, seq_len, is_prompt) in self.graphed_buckets
661
+
662
+ def _is_valid_bucket(self, bucket):
663
+ return bucket[0] * bucket[1] <= self.max_num_batched_tokens
664
+
665
+ def _setup_buckets(self) -> None:
666
+ align_bs = lambda x: min(self.max_num_seqs, x)
667
+ #FIXME: The default values should be max_model_len
668
+ max_prompt_seq = 1024
669
+ max_decode_seq = 2048
670
+ self.bucketing_global_state.prompt_bs_bucket_cfg = read_bucket_settings(
671
+ 'prompt',
672
+ 'bs',
673
+ min=1,
674
+ step=align_bs(32),
675
+ max=self.max_num_prefill_seqs)
676
+ self.bucketing_global_state.decode_bs_bucket_cfg = read_bucket_settings(
677
+ 'decode', 'bs', min=1, step=align_bs(32), max=self.max_num_seqs)
678
+ self.bucketing_global_state.prompt_seq_bucket_cfg = \
679
+ read_bucket_settings(
680
+ 'prompt',
681
+ 'seq',
682
+ min=self.block_size,
683
+ step=self.block_size,
684
+ max=max_prompt_seq)
685
+ self.bucketing_global_state.decode_block_bucket_cfg = \
686
+ read_bucket_settings(
687
+ 'decode',
688
+ 'block',
689
+ min=self.block_size,
690
+ step=self.block_size,
691
+ max=max(self.block_size,
692
+ self.max_num_seqs * max_decode_seq // self.block_size))
693
+ self.graphed_buckets: Set[Any] = set()
694
+
695
+ msg = ("Prompt bucket config (min, step, max_warmup) "
696
+ f"bs:{self.bucketing_global_state.prompt_bs_bucket_cfg}, "
697
+ f"seq:{self.bucketing_global_state.prompt_seq_bucket_cfg}")
698
+ logger.info(msg)
699
+
700
+ msg = ("Decode bucket config (min, step, max_warmup) "
701
+ f"bs:{self.bucketing_global_state.decode_bs_bucket_cfg}, "
702
+ f"block:{self.bucketing_global_state.decode_block_bucket_cfg}")
703
+ logger.info(msg)
704
+
705
+ def _prepare_prompt(
706
+ self,
707
+ seq_group_metadata_list: List[SequenceGroupMetadata],
708
+ ) -> PreparePromptMetadata:
709
+ input_tokens: List[List[int]] = []
710
+ input_positions: List[List[int]] = []
711
+ slot_mapping: List[List[int]] = []
712
+ lora_index_mapping: List[List[int]] = []
713
+ lora_prompt_mapping: List[List[int]] = []
714
+ lora_requests: Set[LoRARequest] = set()
715
+
716
+ seq_lens: List[int] = []
717
+ context_lens: List[int] = []
718
+ query_lens: List[int] = []
719
+ prefix_block_tables: List[List[int]] = []
720
+ multi_modal_inputs_list: List[MultiModalInputs] = []
721
+
722
+ if len(seq_group_metadata_list) == 0:
723
+ return PreparePromptMetadata.empty()
724
+
725
+ for seq_group_metadata in seq_group_metadata_list:
726
+ assert seq_group_metadata.is_prompt
727
+ seq_ids = list(seq_group_metadata.seq_data.keys())
728
+ assert len(seq_ids) == 1
729
+ seq_id = seq_ids[0]
730
+
731
+ computed_block_nums = seq_group_metadata.computed_block_nums
732
+ if (self.scheduler_config is not None
733
+ and self.scheduler_config.chunked_prefill_enabled
734
+ and not (computed_block_nums is None
735
+ or computed_block_nums == [])):
736
+ raise RuntimeError(
737
+ "chunked prefill cannot be used with prefix caching "
738
+ "now.")
739
+
740
+ token_chunk_size = seq_group_metadata.token_chunk_size
741
+ seq_data = seq_group_metadata.seq_data[seq_id]
742
+ context_len = seq_data.get_num_computed_tokens()
743
+ # We should use get_len here because in case of preemption
744
+ # it contains output tokens.
745
+ seq_len = min(seq_data.get_len(), context_len + token_chunk_size)
746
+ prompt_tokens = seq_data.get_token_ids()[context_len:seq_len]
747
+ seq_lens.append(seq_len)
748
+
749
+ # NOTE: This only works for oooooooxxx style attention.
750
+ if computed_block_nums is not None and len(
751
+ computed_block_nums) > 0 and self.sliding_window is None:
752
+ # Prefix is not supported with sliding_window
753
+ context_len = len(computed_block_nums) * self.block_size
754
+ prompt_tokens = prompt_tokens[context_len:]
755
+ prefix_block_tables.append(computed_block_nums)
756
+ elif self.scheduler_config.chunked_prefill_enabled:
757
+ if seq_group_metadata.block_tables is not None:
758
+ # Prefill has chunked before.
759
+ block_table = seq_group_metadata.block_tables[seq_id]
760
+ prefix_block_tables.append(block_table)
761
+ else:
762
+ # The first prefill.
763
+ prefix_block_tables.append([])
764
+ else:
765
+ prefix_block_tables.append([])
766
+ # Right now, prefill start is always 0. However, this
767
+ # assumption can be changed once chunked prefill is introduced.
768
+ assert context_len == 0
769
+
770
+ # actual prompt lens
771
+ context_lens.append(context_len)
772
+ query_lens.append(seq_len - context_len)
773
+ input_tokens.append(prompt_tokens)
774
+ # NOTE(woosuk): Here we assume that the first token in the prompt
775
+ # is always the first token in the sequence.
776
+ input_positions.append(list(range(context_len, seq_len)))
777
+
778
+ mm_data = seq_group_metadata.multi_modal_data
779
+ if mm_data:
780
+ mm_kwargs = self.multi_modal_input_mapper(mm_data)
781
+ multi_modal_inputs_list.append(mm_kwargs)
782
+
783
+ if seq_group_metadata.block_tables is None:
784
+ # During memory profiling, the block tables are not initialized
785
+ # yet. In this case, we just use a dummy slot mapping.
786
+ slot_mapping.append([_PAD_SLOT_ID] * seq_len)
787
+ continue
788
+
789
+ # Compute the slot mapping.
790
+ slot_mapping.append([])
791
+ block_table = seq_group_metadata.block_tables[seq_id]
792
+
793
+ # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
794
+ # where start_idx is max(0, seq_len - sliding_window).
795
+ # For example, if the prompt len is 10, sliding window is 8, and
796
+ # block size is 4, the first two tokens are masked and the slot
797
+ # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
798
+ start_idx = 0
799
+ if self.sliding_window is not None:
800
+ assert context_len == 0, (
801
+ "Prefix caching is currently not supported with "
802
+ "sliding window attention")
803
+ start_idx = max(0, seq_len - self.sliding_window)
804
+ for i in range(context_len, seq_len):
805
+ if i < start_idx:
806
+ slot_mapping[-1].append(_PAD_SLOT_ID)
807
+ continue
808
+
809
+ block_number = block_table[i // self.block_size]
810
+ block_offset = i % self.block_size
811
+ slot = block_number * self.block_size + block_offset
812
+ slot_mapping[-1].append(slot)
813
+
814
+ max_query_len = max(query_lens)
815
+ sum_query_len = sum(query_lens)
816
+ real_num_seqs = len(query_lens)
817
+ assert max_query_len > 0
818
+
819
+ max_prompt_len = max(
820
+ find_bucket(max(seq_lens),
821
+ self.bucketing_global_state.prompt_seq_bucket_cfg),
822
+ self.block_size)
823
+
824
+ lora_ids: List[int] = []
825
+ for seq_group_metadata, context_len in zip(seq_group_metadata_list,
826
+ context_lens):
827
+ lora_id = seq_group_metadata.lora_int_id
828
+ lora_ids.append(lora_id)
829
+
830
+ if lora_id > 0:
831
+ lora_requests.add(seq_group_metadata.lora_request)
832
+
833
+ lora_index_mapping += [lora_id] * (max_prompt_len - context_len)
834
+ lora_prompt_mapping.extend(
835
+ [lora_id] *
836
+ (max_prompt_len - context_len
837
+ if seq_group_metadata.sampling_params.prompt_logprobs else 1))
838
+
839
+ input_tokens = make_tensor_with_pad(input_tokens,
840
+ max_len=max_prompt_len,
841
+ pad=0,
842
+ dtype=torch.long,
843
+ device=self.device)
844
+
845
+ input_positions = make_tensor_with_pad(input_positions,
846
+ max_len=max_prompt_len,
847
+ pad=0,
848
+ dtype=torch.long,
849
+ device=self.device)
850
+
851
+ slot_mapping = make_tensor_with_pad(slot_mapping,
852
+ max_len=max_prompt_len,
853
+ pad=_PAD_SLOT_ID,
854
+ dtype=torch.long,
855
+ device=self.device)
856
+
857
+ seq_lens_tensor = torch.tensor(seq_lens,
858
+ dtype=torch.long,
859
+ device=self.device)
860
+
861
+ block_indices, block_offsets = precompute_indices_and_offsets(
862
+ self.block_size, slot_mapping, True)
863
+ attn_metadata = self.attn_backend.make_metadata(
864
+ is_prompt=True,
865
+ block_list=None,
866
+ block_mapping=None,
867
+ block_usage=None,
868
+ block_indices=block_indices,
869
+ block_offsets=block_offsets,
870
+ block_scales=None,
871
+ attn_bias=None,
872
+ seq_lens_tensor=seq_lens_tensor,
873
+ num_prefills=real_num_seqs,
874
+ num_prefill_tokens=sum_query_len,
875
+ num_decode_tokens=0,
876
+ slot_mapping=slot_mapping,
877
+ multi_modal_placeholder_index_maps=
878
+ None # FIXME(kzawora): mutli-modality will not work here
879
+ )
880
+ multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
881
+
882
+ return PreparePromptMetadata(input_tokens=input_tokens,
883
+ input_positions=input_positions,
884
+ attn_metadata=attn_metadata,
885
+ seq_lens=seq_lens,
886
+ query_lens=query_lens,
887
+ lora_index_mapping=lora_index_mapping,
888
+ lora_prompt_mapping=lora_prompt_mapping,
889
+ lora_requests=lora_requests,
890
+ multi_modal_kwargs=multi_modal_kwargs,
891
+ slot_mapping=slot_mapping,
892
+ lora_ids=lora_ids)
893
+
894
+ def _prepare_decode(
895
+ self,
896
+ seq_group_metadata_list: List[SequenceGroupMetadata],
897
+ ) -> PrepareDecodeMetadata:
898
+ input_tokens: List[List[int]] = []
899
+ input_positions: List[List[int]] = []
900
+ slot_mapping: List[List[int]] = []
901
+ seq_lens: List[int] = []
902
+ block_tables: List[List[int]] = []
903
+ lora_index_mapping: List[List[int]] = []
904
+ lora_prompt_mapping: List[List[int]] = []
905
+ lora_requests: Set[LoRARequest] = set()
906
+
907
+ if len(seq_group_metadata_list) == 0:
908
+ return PrepareDecodeMetadata.empty()
909
+ lora_ids: List[int] = []
910
+
911
+ dummy_slots = itertools.cycle(
912
+ range(_PAD_SLOT_ID, _PAD_SLOT_ID + self.block_size))
913
+
914
+ for seq_group_metadata in seq_group_metadata_list:
915
+ assert not seq_group_metadata.is_prompt
916
+ assert seq_group_metadata.token_chunk_size == 1
917
+
918
+ seq_ids = list(seq_group_metadata.seq_data.keys())
919
+ lora_id = seq_group_metadata.lora_int_id
920
+ lora_ids.append(lora_id)
921
+
922
+ if lora_id > 0:
923
+ lora_requests.add(seq_group_metadata.lora_request)
924
+
925
+ for seq_id in seq_ids:
926
+ seq_data = seq_group_metadata.seq_data[seq_id]
927
+ generation_token = seq_data.get_last_token_id()
928
+ input_tokens.append([generation_token])
929
+
930
+ seq_len = seq_data.get_len()
931
+ position = seq_len - 1
932
+ input_positions.append([position])
933
+
934
+ seq_len = seq_len if self.sliding_window is None else min(
935
+ seq_len, self.sliding_window)
936
+ seq_lens.append(seq_len)
937
+
938
+ block_table = seq_group_metadata.block_tables[seq_id]
939
+ if len(block_table) == 0:
940
+ block_number = _PAD_BLOCK_ID
941
+ else:
942
+ block_number = block_table[position // self.block_size]
943
+ if block_number == _PAD_BLOCK_ID:
944
+ slot = next(dummy_slots)
945
+ else:
946
+ block_offset = position % self.block_size
947
+ slot = block_number * self.block_size + block_offset
948
+ slot_mapping.append([slot])
949
+ lora_index_mapping.append(lora_id)
950
+ lora_prompt_mapping.append(lora_id)
951
+
952
+ if self.sliding_window is not None:
953
+ sliding_window_blocks = (self.sliding_window //
954
+ self.block_size)
955
+ block_table = block_table[-sliding_window_blocks:]
956
+ block_tables.append(block_table)
957
+
958
+ input_tokens = torch.tensor(input_tokens,
959
+ dtype=torch.long,
960
+ device=self.device)
961
+ input_positions = torch.tensor(input_positions,
962
+ dtype=torch.long,
963
+ device=self.device)
964
+
965
+ num_decode_tokens = sum(seq_lens)
966
+
967
+ blocks_used = [len(bt) for bt in block_tables if bt]
968
+ block_list = []
969
+ block_scales = []
970
+ for i, bt in enumerate(block_tables):
971
+ block_list.extend(bt)
972
+ blocks_in_group = len(bt)
973
+ if blocks_in_group > 0:
974
+ scale = 1.0 / blocks_in_group
975
+ block_scales.extend([scale] * blocks_in_group)
976
+
977
+ block_mapping_nested: List[List[int]] = [
978
+ [i] * b_u for i, b_u in enumerate(blocks_used)
979
+ ]
980
+ block_mapping: List[int] = list(
981
+ itertools.chain.from_iterable(block_mapping_nested))
982
+
983
+ last_block = [
984
+ sl % self.block_size + 1 for sl in itertools.chain(*slot_mapping)
985
+ ]
986
+ block_usage = [[self.block_size] * (b_u - 1) + [lb]
987
+ for b_u, lb in zip(blocks_used, last_block)]
988
+ block_usage = list(itertools.chain(*block_usage))
989
+
990
+ block_bucket_size = find_bucket(
991
+ len(block_list),
992
+ self.bucketing_global_state.decode_block_bucket_cfg)
993
+ block_list = pad_list(block_list, block_bucket_size, _PAD_BLOCK_ID)
994
+ block_mapping = pad_list(block_mapping, block_bucket_size, -1)
995
+ block_usage = pad_list(block_usage, block_bucket_size, 1)
996
+ block_scales = pad_list(block_scales, block_bucket_size, 0.0)
997
+
998
+ block_list = torch.tensor(block_list,
999
+ dtype=torch.int,
1000
+ device=self.device)
1001
+ block_mapping = torch.tensor(block_mapping,
1002
+ dtype=torch.long,
1003
+ device=self.device)
1004
+ block_usage = torch.tensor(block_usage,
1005
+ dtype=self.model_config.dtype,
1006
+ device=self.device)
1007
+
1008
+ slot_mapping = torch.tensor(slot_mapping,
1009
+ dtype=torch.long,
1010
+ device=self.device)
1011
+
1012
+ block_indices, block_offsets = precompute_indices_and_offsets(
1013
+ self.block_size, slot_mapping, False)
1014
+ block_scales = torch.tensor(block_scales,
1015
+ dtype=self.model_config.dtype,
1016
+ device=self.device)
1017
+
1018
+ attn_metadata = self.attn_backend.make_metadata(
1019
+ is_prompt=False,
1020
+ block_list=block_list,
1021
+ block_mapping=block_mapping,
1022
+ block_usage=block_usage,
1023
+ block_indices=block_indices,
1024
+ block_offsets=block_offsets,
1025
+ block_scales=block_scales,
1026
+ attn_bias=None,
1027
+ seq_lens_tensor=None,
1028
+ num_prefills=0,
1029
+ num_prefill_tokens=0,
1030
+ num_decode_tokens=num_decode_tokens,
1031
+ slot_mapping=slot_mapping,
1032
+ multi_modal_placeholder_index_maps=None)
1033
+ return PrepareDecodeMetadata(input_tokens=input_tokens,
1034
+ input_positions=input_positions,
1035
+ attn_metadata=attn_metadata,
1036
+ lora_index_mapping=lora_index_mapping,
1037
+ lora_prompt_mapping=lora_prompt_mapping,
1038
+ lora_requests=lora_requests,
1039
+ slot_mapping=slot_mapping,
1040
+ lora_ids=lora_ids)
1041
+
1042
+ def prepare_input_tensors(
1043
+ self,
1044
+ seq_group_metadata_list: List[SequenceGroupMetadata],
1045
+ ) -> Tuple[TModelInputForHPU, SamplingMetadata]:
1046
+ if len(seq_group_metadata_list) == 0:
1047
+ return self._model_input_cls(), None
1048
+
1049
+ input_tokens = None
1050
+ input_positions = None
1051
+ lora_mapping = None
1052
+ lora_requests = None
1053
+ multi_modal_kwargs = None
1054
+ batch_type = None
1055
+ seq_lens = None
1056
+ query_lens = None
1057
+ real_batch_size = None
1058
+ batch_size_padded = None
1059
+
1060
+ self.event_start = self.profiler.get_timestamp_us()
1061
+ is_prompt = seq_group_metadata_list[0].is_prompt
1062
+ base_event_name = 'prompt' if is_prompt else 'decode'
1063
+ self.profiler.start('internal', base_event_name)
1064
+
1065
+ real_batch_size = len(seq_group_metadata_list)
1066
+ bucket_cfg = self.bucketing_global_state.prompt_bs_bucket_cfg \
1067
+ if is_prompt else self.bucketing_global_state.decode_bs_bucket_cfg
1068
+ batch_size_padded = find_bucket(real_batch_size, bucket_cfg)
1069
+ batch_size_padding = batch_size_padded - real_batch_size
1070
+ seq_group_metadata_list = seq_group_metadata_list.copy()
1071
+ if batch_size_padding > 0:
1072
+ dummy_seq_group_metadata = self.create_dummy_seq_group_metadata(
1073
+ 0, 0, is_prompt)
1074
+ seq_group_metadata_list.extend(dummy_seq_group_metadata
1075
+ for _ in range(batch_size_padding))
1076
+
1077
+ prefill_reqs = []
1078
+ decode_reqs = []
1079
+ for seq_group_meta in seq_group_metadata_list:
1080
+ if seq_group_meta.is_prompt:
1081
+ prefill_reqs.append(seq_group_meta)
1082
+ else:
1083
+ decode_reqs.append(seq_group_meta)
1084
+
1085
+ # Prepare input tensors.
1086
+ (
1087
+ input_tokens,
1088
+ input_positions,
1089
+ prefill_attn_metadata,
1090
+ seq_lens,
1091
+ query_lens,
1092
+ lora_index_mapping,
1093
+ lora_prompt_mapping,
1094
+ lora_requests,
1095
+ multi_modal_kwargs,
1096
+ slot_mapping,
1097
+ lora_ids,
1098
+ ) = self._prepare_prompt(prefill_reqs)
1099
+ (
1100
+ decode_input_tokens,
1101
+ decode_input_positions,
1102
+ decode_attn_metadata,
1103
+ decode_lora_index_mapping,
1104
+ decode_lora_prompt_mapping,
1105
+ decode_lora_requests,
1106
+ decode_slot_mapping,
1107
+ decode_lora_ids,
1108
+ ) = self._prepare_decode(decode_reqs)
1109
+ sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
1110
+ seq_lens, query_lens,
1111
+ self.device,
1112
+ self.pin_memory)
1113
+
1114
+ if not self.scheduler_config.chunked_prefill_enabled:
1115
+ assert (len(prefill_reqs) and len(decode_reqs)) == 0
1116
+
1117
+ num_prefills = len(seq_lens)
1118
+ num_prefill_tokens = len(input_tokens)
1119
+ num_decode_tokens = len(decode_input_tokens)
1120
+
1121
+ # NOTE(kzawora): Here we diverge from GPU code - we don't
1122
+ # support mixed batches, so we either use decode or prefill
1123
+ # inputs, without coalescing.
1124
+ assert (num_prefills == 0 and num_decode_tokens > 0) or (
1125
+ num_prefills > 0
1126
+ and num_decode_tokens == 0), "HPU does not support mixed batches!"
1127
+ if num_decode_tokens > 0:
1128
+ input_tokens = decode_input_tokens
1129
+ input_positions = decode_input_positions
1130
+ slot_mapping = decode_slot_mapping
1131
+ lora_index_mapping = decode_lora_index_mapping
1132
+ lora_prompt_mapping = decode_lora_prompt_mapping
1133
+ lora_requests = decode_lora_requests
1134
+ lora_ids = decode_lora_ids
1135
+
1136
+ # FIXME: We need to adjust selected_token_indices to accommodate
1137
+ # for padding
1138
+ max_len = input_tokens.size(1)
1139
+ paddings = [max_len - s for s in seq_lens]
1140
+ paddings = [0] + paddings[:-1]
1141
+ paddings = list(itertools.accumulate(paddings))
1142
+ paddings_prompt_logprobs = []
1143
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
1144
+ if seq_group_metadata.sampling_params.prompt_logprobs is not None \
1145
+ and seq_group_metadata.is_prompt:
1146
+ paddings_prompt_logprobs += ([paddings[i]] * seq_lens[i])
1147
+ paddings = torch.tensor(
1148
+ paddings_prompt_logprobs if paddings_prompt_logprobs else paddings,
1149
+ dtype=sampling_metadata.selected_token_indices.dtype,
1150
+ device=sampling_metadata.selected_token_indices.device)
1151
+ sampling_metadata.selected_token_indices.add_(paddings)
1152
+
1153
+ if self.lora_config:
1154
+ lora_mapping = LoRAMapping(
1155
+ **dict(index_mapping=lora_index_mapping,
1156
+ prompt_mapping=lora_prompt_mapping,
1157
+ is_prefill=(num_prefills > 0)))
1158
+ else:
1159
+ lora_mapping = None
1160
+
1161
+ if (prefill_attn_metadata is not None
1162
+ and decode_attn_metadata is not None):
1163
+ batch_type = BatchType.MIXED
1164
+ raise NotImplementedError("Mixed batch is not supported on HPU")
1165
+ elif prefill_attn_metadata is not None:
1166
+ batch_type = BatchType.PREFILL
1167
+ else:
1168
+ batch_type = BatchType.DECODE
1169
+
1170
+ metadata_dict = {
1171
+ "input_tokens": input_tokens,
1172
+ "input_positions": input_positions,
1173
+ "selected_token_indices": sampling_metadata.selected_token_indices,
1174
+ "lora_requests": lora_requests,
1175
+ "lora_mapping": lora_mapping,
1176
+ "multi_modal_kwargs": multi_modal_kwargs,
1177
+ "num_prefill_tokens": num_prefill_tokens,
1178
+ "num_decode_tokens": num_decode_tokens,
1179
+ "slot_mapping": slot_mapping,
1180
+ "num_prefills": num_prefills,
1181
+ "batch_type": batch_type,
1182
+ "seq_lens": seq_lens,
1183
+ "query_lens": query_lens
1184
+ }
1185
+ if prefill_attn_metadata is not None:
1186
+ metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
1187
+ else:
1188
+ assert decode_attn_metadata is not None
1189
+ metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
1190
+
1191
+ attn_metadata = prefill_attn_metadata if \
1192
+ prefill_attn_metadata is not None else decode_attn_metadata
1193
+
1194
+ return self._model_input_cls(input_tokens=input_tokens,
1195
+ seq_lens=seq_lens,
1196
+ query_lens=query_lens,
1197
+ input_positions=input_positions,
1198
+ attn_metadata=attn_metadata,
1199
+ lora_requests=lora_requests,
1200
+ lora_mapping=lora_mapping,
1201
+ multi_modal_kwargs=multi_modal_kwargs,
1202
+ real_batch_size=real_batch_size,
1203
+ batch_size_padded=batch_size_padded,
1204
+ lora_ids=lora_ids), \
1205
+ sampling_metadata
1206
+
1207
+ def _seq_len(self, attn_metadata):
1208
+ if attn_metadata.num_prefills != 0:
1209
+ return attn_metadata.slot_mapping.size(1)
1210
+ else:
1211
+ return attn_metadata.block_list.numel()
1212
+
1213
+ def trim_attn_metadata(self, metadata: AttentionMetadata) -> object:
1214
+ # NOTE(kzawora): To anyone working on this in the future:
1215
+ # Trimming metadata is required when using HPUGraphs.
1216
+ # Attention metadata is going to be hashed by PT bridge, and
1217
+ # appropriate HPUGraphs will be matched based on all inputs' hash.
1218
+
1219
+ # Before you put more keys in here, make sure you know their
1220
+ # value type and make sure you know how it's going to be hashed.
1221
+ # You can find that information in input_hash function
1222
+ # in habana_frameworks/torch/hpu/graphs.py. You can also hash
1223
+ # it manually with torch.hpu.graphs.input_hash(attention_metadata)
1224
+
1225
+ # If you use primitive types here - they will get hashed based
1226
+ # on their value. You *will* get lots of excessive graph captures
1227
+ # (and an OOM eventually) if you decide to put something like
1228
+ # seq_len int here.
1229
+ # If you absolutely need a scalar, put it in a tensor. Tensors
1230
+ # get hashed using their metadata, not their values:
1231
+ # input_hash(torch.tensor(123)) == input_hash(torch.tensor(321))
1232
+ # input_hash(123) != input_hash(321)
1233
+ # input_hash("abc") != input_hash("cba")
1234
+ attention_metadata = subtuple(metadata, 'TrimmedAttentionMetadata', [
1235
+ 'attn_bias', 'seq_lens_tensor', 'block_list', 'block_mapping',
1236
+ 'block_usage', 'slot_mapping', 'is_prompt', 'block_indices',
1237
+ 'block_offsets', 'block_scales'
1238
+ ])
1239
+ return attention_metadata
1240
+
1241
+ def create_dummy_seq_group_metadata(self,
1242
+ group_id,
1243
+ seq_len,
1244
+ is_prompt,
1245
+ lora_request=None):
1246
+ sampling_params = SamplingParams(temperature=0)
1247
+ num_blocks = math.ceil(seq_len / self.block_size)
1248
+ seq_len = max(seq_len, 1)
1249
+ if is_prompt:
1250
+ input_len = seq_len
1251
+ output_len = 0
1252
+ block_tables = None
1253
+ else:
1254
+ input_len = seq_len - 1
1255
+ output_len = 1
1256
+ block_tables = {group_id: [_PAD_BLOCK_ID] * num_blocks}
1257
+ prompt_token_ids = [0] * input_len
1258
+ output_token_ids = [1] * output_len
1259
+ prompt_token_ids_array = array('l', prompt_token_ids) # noqa: F821
1260
+ seq_data = SequenceData(prompt_token_ids_array)
1261
+ seq_data.output_token_ids = output_token_ids
1262
+ return SequenceGroupMetadata(request_id=str(group_id),
1263
+ is_prompt=(output_len == 0),
1264
+ seq_data={group_id: seq_data},
1265
+ sampling_params=sampling_params,
1266
+ block_tables=block_tables,
1267
+ lora_request=lora_request)
1268
+
1269
+ def profile_run(self) -> None:
1270
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
1271
+ kv_caches = [None] * num_layers
1272
+ max_batch_size = self.bucketing_global_state.prompt_bs_bucket_cfg[-1]
1273
+ max_seq_len = min(
1274
+ self.bucketing_global_state.prompt_seq_bucket_cfg[-1],
1275
+ self.max_num_batched_tokens // max_batch_size)
1276
+
1277
+ self.warmup_scenario(max_batch_size, max_seq_len, True, kv_caches,
1278
+ False, True)
1279
+ return
1280
+
1281
+ def warmup_scenario(self,
1282
+ batch_size,
1283
+ seq_len,
1284
+ is_prompt,
1285
+ kv_caches,
1286
+ is_pt_profiler_run=False,
1287
+ is_lora_profile_run=False) -> None:
1288
+ use_graphs = self._use_graphs(batch_size, seq_len, is_prompt)
1289
+ scenario_name = ("warmup_"
1290
+ f"{'prompt' if is_prompt else 'decode'}_"
1291
+ f"bs{batch_size}_"
1292
+ f"seq{seq_len}_"
1293
+ f"graphs{'T' if use_graphs else 'F'}")
1294
+ max_num_seqs = self.scheduler_config.max_num_seqs
1295
+ # This represents the maximum number of different requests
1296
+ # that will have unique loras, an therefore the max amount of memory
1297
+ # consumption create dummy lora request copies from the lora request
1298
+ # passed in, which contains a lora from the lora warmup path.
1299
+ dummy_lora_requests: List[LoRARequest] = []
1300
+ dummy_lora_requests_per_seq: List[LoRARequest] = []
1301
+ if self.lora_config and is_lora_profile_run:
1302
+ assert self.lora_manager is not None
1303
+ with self.lora_manager.dummy_lora_cache():
1304
+ for idx in range(self.lora_config.max_loras):
1305
+ lora_id = idx + 1
1306
+ dummy_lora_request = LoRARequest(
1307
+ lora_name=f"warmup_{lora_id}",
1308
+ lora_int_id=lora_id,
1309
+ lora_local_path="/not/a/real/path",
1310
+ )
1311
+ self.lora_manager.add_dummy_lora(dummy_lora_request,
1312
+ rank=LORA_WARMUP_RANK)
1313
+ dummy_lora_requests.append(dummy_lora_request)
1314
+ dummy_lora_requests_per_seq = [
1315
+ dummy_lora_requests[idx % len(dummy_lora_requests)]
1316
+ for idx in range(max_num_seqs)
1317
+ ]
1318
+ self.profiler.start('internal', scenario_name)
1319
+ times = 3 if use_graphs or is_pt_profiler_run else 1
1320
+ if self.lora_config and not is_lora_profile_run:
1321
+ lora_mapping = LoRAMapping(
1322
+ **dict(index_mapping=[0] * batch_size * seq_len,
1323
+ prompt_mapping=[0] * batch_size * seq_len,
1324
+ is_prefill=is_prompt))
1325
+ self.set_active_loras(set(), lora_mapping)
1326
+ if is_prompt:
1327
+ seqs = [
1328
+ self.create_dummy_seq_group_metadata(
1329
+ i,
1330
+ seq_len,
1331
+ is_prompt,
1332
+ lora_request=dummy_lora_requests_per_seq[i]
1333
+ if dummy_lora_requests_per_seq else None)
1334
+ for i in range(batch_size)
1335
+ ]
1336
+ else:
1337
+ # FIXME: seq_len is actually number of blocks
1338
+ blocks = [seq_len // batch_size for _ in range(batch_size)]
1339
+ blocks[0] += seq_len % batch_size
1340
+ seqs = [
1341
+ self.create_dummy_seq_group_metadata(
1342
+ i,
1343
+ b * self.block_size - 1,
1344
+ is_prompt,
1345
+ lora_request=dummy_lora_requests_per_seq[i]
1346
+ if dummy_lora_requests_per_seq else None)
1347
+ for i, b in enumerate(blocks)
1348
+ ]
1349
+ torch.hpu.synchronize()
1350
+ profiler = None
1351
+ if is_pt_profiler_run and self.is_driver_worker:
1352
+ profiler = setup_profiler()
1353
+ profiler.start()
1354
+ for _ in range(times):
1355
+ inputs = self.prepare_model_input(seqs)
1356
+ self.execute_model(inputs, kv_caches, warmup_mode=True)
1357
+ torch.hpu.synchronize()
1358
+ if profiler:
1359
+ profiler.step()
1360
+ if profiler:
1361
+ profiler.stop()
1362
+ self.profiler.end()
1363
+ gc.collect()
1364
+
1365
+ def remove_all_loras(self):
1366
+ if not self.lora_manager:
1367
+ raise RuntimeError("LoRA is not enabled.")
1368
+ self.lora_manager.remove_all_adapters()
1369
+
1370
+ def set_active_loras(self, lora_requests: Set[LoRARequest],
1371
+ lora_mapping: LoRAMapping) -> None:
1372
+ if not self.lora_manager:
1373
+ raise RuntimeError("LoRA is not enabled.")
1374
+ self.lora_manager.set_active_adapters(lora_requests, lora_mapping)
1375
+
1376
+ def add_lora(self, lora_request: LoRARequest) -> bool:
1377
+ if not self.lora_manager:
1378
+ raise RuntimeError("LoRA is not enabled.")
1379
+ return self.lora_manager.add_adapter(lora_request)
1380
+
1381
+ def remove_lora(self, lora_id: int) -> bool:
1382
+ if not self.lora_manager:
1383
+ raise RuntimeError("LoRA is not enabled.")
1384
+ return self.lora_manager.remove_adapter(lora_id)
1385
+
1386
+ def pin_lora(self, lora_id: int) -> bool:
1387
+ if not self.lora_manager:
1388
+ raise RuntimeError("LoRA is not enabled.")
1389
+ return self.lora_manager.pin_adapter(lora_id)
1390
+
1391
+ def list_loras(self) -> Set[int]:
1392
+ if not self.lora_manager:
1393
+ raise RuntimeError("LoRA is not enabled.")
1394
+ return self.lora_manager.list_adapters()
1395
+
1396
+ def log_warmup(self, phase, i, max_i, batch_size, seq_len):
1397
+ free_mem = format_bytes(
1398
+ HabanaMemoryProfiler.current_free_device_memory())
1399
+ dim = "num_blocks"
1400
+ if phase == "Prompt":
1401
+ dim = "seq_len"
1402
+ msg = (f"[Warmup][{phase}][{i+1}/{max_i}] "
1403
+ f"batch_size:{batch_size} "
1404
+ f"{dim}:{seq_len} "
1405
+ f"free_mem:{free_mem}")
1406
+ logger.info(msg)
1407
+
1408
+ def warmup_all_buckets(self, buckets, is_prompt, kv_caches):
1409
+ for i, (batch_size, seq_len) in enumerate(reversed(buckets)):
1410
+ self.log_warmup('Prompt' if is_prompt else 'Decode', i,
1411
+ len(buckets), batch_size, seq_len)
1412
+ self.warmup_scenario(batch_size, seq_len, is_prompt, kv_caches)
1413
+
1414
+ def warmup_graphs(self,
1415
+ strategy,
1416
+ buckets,
1417
+ is_prompt,
1418
+ kv_caches,
1419
+ available_mem,
1420
+ starting_mem=0,
1421
+ total_batch_seq=0.001):
1422
+ total_mem = starting_mem
1423
+ idx = 0
1424
+ phase = f'Graph/{"Prompt" if is_prompt else "Decode"}'
1425
+ num_candidates = len(buckets)
1426
+ ordering : Union[Callable[[Any], Tuple[Any, Any]], \
1427
+ Callable[[Any], Tuple[Any, Any, Any]]]
1428
+ if strategy == 'min_tokens':
1429
+ ordering = lambda b: (b[0] * b[1], b[1], b[0])
1430
+ elif strategy == 'max_bs':
1431
+ ordering = lambda b: (-b[0], b[1])
1432
+ else:
1433
+ raise NotImplementedError(
1434
+ f'Unsupported graph allocation strategy: {strategy}')
1435
+ buckets = list(sorted(buckets, key=ordering))
1436
+ captured_all = True
1437
+ for idx, (batch_size, seq_len) in enumerate(buckets):
1438
+ # Graph memory usage is proportional to seq dimension in a batch
1439
+ batch_seq = batch_size * seq_len if is_prompt else batch_size
1440
+ mem_estimate = batch_seq / total_batch_seq * total_mem
1441
+ if mem_estimate >= available_mem:
1442
+ captured_all = False
1443
+ continue
1444
+ graphed_bucket = (batch_size, seq_len, is_prompt)
1445
+ if graphed_bucket in self.graphed_buckets:
1446
+ continue
1447
+ self.graphed_buckets.add(graphed_bucket)
1448
+ self.log_warmup(phase, idx, num_candidates, batch_size, seq_len)
1449
+ with HabanaMemoryProfiler() as mem_prof:
1450
+ self.warmup_scenario(batch_size, seq_len, is_prompt, kv_caches)
1451
+ used_mem = align_workers(mem_prof.consumed_device_memory,
1452
+ torch.distributed.ReduceOp.MAX)
1453
+ available_mem -= used_mem
1454
+ total_mem += used_mem
1455
+ total_batch_seq += batch_seq
1456
+
1457
+ return total_mem, total_batch_seq, captured_all
1458
+
1459
+ def log_graph_warmup_summary(self, buckets, is_prompt, total_mem):
1460
+ num_candidates = len(buckets)
1461
+ phase = f'Graph/{"Prompt" if is_prompt else "Decode"}'
1462
+ graphed = list(c[:2] for c in self.graphed_buckets
1463
+ if c[2] == is_prompt)
1464
+ if num_candidates == 0:
1465
+ num_candidates = 1
1466
+ msg = (f'{phase} captured:{len(graphed)} '
1467
+ f'({100 * len(graphed) / num_candidates:.1f}%) '
1468
+ f'used_mem:{format_bytes(total_mem)} '
1469
+ f'buckets:{sorted(list(graphed))}')
1470
+ logger.info(msg)
1471
+
1472
+ @torch.inference_mode()
1473
+ def warmup_model(self, kv_caches: List[torch.Tensor]) -> None:
1474
+ if profile := os.environ.get('VLLM_PT_PROFILE', None):
1475
+ phase, bs, seq_len, graph = profile.split('_')
1476
+ is_prompt = phase == 'prompt'
1477
+ graphs = graph == 't'
1478
+ if graphs:
1479
+ self.graphed_buckets.add((int(bs), int(seq_len), is_prompt))
1480
+ self.warmup_scenario(int(bs), int(seq_len), is_prompt, kv_caches,
1481
+ True)
1482
+ raise AssertionError("Finished profiling")
1483
+ if self.skip_warmup:
1484
+ logger.info("Skipping warmup...")
1485
+ return
1486
+ self.profiler.start('internal', 'warmup')
1487
+ max_blocks = kv_caches[0][0].size(0)
1488
+
1489
+ self.bucketing_global_state.prompt_buckets, prompt_omitted_buckets = \
1490
+ generate_prompt_buckets(
1491
+ self.bucketing_global_state.prompt_bs_bucket_cfg,
1492
+ self.bucketing_global_state.prompt_seq_bucket_cfg,
1493
+ self.max_num_batched_tokens)
1494
+
1495
+ msg = (f"Generated {len(self.bucketing_global_state.prompt_buckets)} "
1496
+ f"prompt buckets [bs, seq]: \
1497
+ {list(sorted(self.bucketing_global_state.prompt_buckets))}")
1498
+ logger.info(msg)
1499
+
1500
+ msg = (f"Omitted {len(prompt_omitted_buckets)} "
1501
+ "prompt buckets due to exceeded token budget "
1502
+ f"(max_num_batched_tokens={self.max_num_batched_tokens})")
1503
+ logger.info(msg)
1504
+
1505
+ msg = f"Omitted prompt buckets: {list(sorted(prompt_omitted_buckets))}"
1506
+ logger.debug(msg)
1507
+
1508
+ self.bucketing_global_state.decode_buckets = generate_decode_buckets(
1509
+ self.bucketing_global_state.decode_bs_bucket_cfg,
1510
+ self.bucketing_global_state.decode_block_bucket_cfg, max_blocks)
1511
+ logger.info("Generated %d decode buckets [bs, total_blocks]: %s",
1512
+ len(self.bucketing_global_state.decode_buckets),
1513
+ list(sorted(self.bucketing_global_state.decode_buckets)))
1514
+
1515
+ if not htorch.utils.internal.is_lazy() and not self.enforce_eager:
1516
+ cache_size_limit = len(
1517
+ self.bucketing_global_state.prompt_buckets) + len(
1518
+ self.bucketing_global_state.decode_buckets) + 1
1519
+ torch._dynamo.config.cache_size_limit = max(
1520
+ cache_size_limit, torch._dynamo.config.cache_size_limit)
1521
+ # Multiply by 8 to follow the original default ratio between
1522
+ # the cache_size_limit and accumulated_cache_size_limit
1523
+ torch._dynamo.config.accumulated_cache_size_limit = max(
1524
+ cache_size_limit * 8,
1525
+ torch._dynamo.config.accumulated_cache_size_limit)
1526
+
1527
+ start_mem = HabanaMemoryProfiler.current_device_memory_usage()
1528
+ start_time = time.perf_counter()
1529
+
1530
+ compile_only_mode_context = functools.partial(bc.env_setting,
1531
+ "PT_COMPILE_ONLY_MODE",
1532
+ True)
1533
+ can_use_compile_only_mode = True
1534
+ try:
1535
+ with compile_only_mode_context():
1536
+ pass
1537
+ logger.debug("Using PT_COMPILE_ONLY_MODE.")
1538
+ except KeyError:
1539
+ can_use_compile_only_mode = False
1540
+ logger.warning('Cannot use PT_COMPILE_ONLY_MODE. '
1541
+ 'Warmup time will be negatively impacted. '
1542
+ 'Please update Gaudi Software Suite.')
1543
+ with compile_only_mode_context(
1544
+ ) if can_use_compile_only_mode else contextlib.nullcontext():
1545
+ self.warmup_all_buckets(self.bucketing_global_state.prompt_buckets,
1546
+ True, kv_caches)
1547
+ self.warmup_all_buckets(self.bucketing_global_state.decode_buckets,
1548
+ False, kv_caches)
1549
+
1550
+ if not self.enforce_eager and htorch.utils.internal.is_lazy():
1551
+ assert self.mem_margin is not None, \
1552
+ ("HabanaWorker.determine_num_available_blocks needs "
1553
+ "to be called before warming up the model.")
1554
+ free_mem = HabanaMemoryProfiler.current_free_device_memory()
1555
+ graph_free_mem = free_mem - self.mem_margin
1556
+ graph_free_mem = align_workers(graph_free_mem,
1557
+ torch.distributed.ReduceOp.MIN)
1558
+ prompt_graph_mem_ratio = float(
1559
+ os.environ.get('VLLM_GRAPH_PROMPT_RATIO', '0.3'))
1560
+ prompt_available_memory = (prompt_graph_mem_ratio *
1561
+ graph_free_mem)
1562
+ decode_available_memory = (graph_free_mem -
1563
+ prompt_available_memory)
1564
+ msg = (
1565
+ f"Using {format_bytes(graph_free_mem)}"
1566
+ f"/{format_bytes(free_mem)} "
1567
+ "of free device memory for HPUGraphs, "
1568
+ f"{format_bytes(prompt_available_memory)} for prompt and "
1569
+ f"{format_bytes(decode_available_memory)} for decode "
1570
+ f"(VLLM_GRAPH_PROMPT_RATIO={prompt_graph_mem_ratio})")
1571
+ logger.info(msg)
1572
+ prompt_strategy = os.environ.get('VLLM_GRAPH_PROMPT_STRATEGY',
1573
+ 'min_tokens')
1574
+ decode_strategy = os.environ.get('VLLM_GRAPH_DECODE_STRATEGY',
1575
+ 'max_bs')
1576
+ mem_post_prompt, prompt_batch_seq, prompt_captured_all = \
1577
+ self.warmup_graphs(
1578
+ prompt_strategy, self.bucketing_global_state.prompt_buckets,
1579
+ True, kv_caches, prompt_available_memory)
1580
+ mem_post_decode, decode_batch_seq, decode_captured_all = \
1581
+ self.warmup_graphs(
1582
+ decode_strategy, self.bucketing_global_state.decode_buckets,
1583
+ False, kv_caches, decode_available_memory)
1584
+
1585
+ # Not all prompt buckets were captured, but all decode buckets
1586
+ # were captured and we have some free graph-allocated space
1587
+ # left. Let's try to use it for capturing more prompt buckets.
1588
+ if (mem_post_decode + mem_post_prompt < graph_free_mem
1589
+ and not prompt_captured_all and decode_captured_all):
1590
+ mem_post_prompt, _, prompt_captured_all = (
1591
+ self.warmup_graphs(
1592
+ prompt_strategy,
1593
+ self.bucketing_global_state.prompt_buckets, True,
1594
+ kv_caches,
1595
+ graph_free_mem - mem_post_prompt - mem_post_decode,
1596
+ mem_post_prompt, prompt_batch_seq))
1597
+
1598
+ # Not all decode buckets were captured, but all prompt buckets
1599
+ # were captured and we have some free graph-allocated space
1600
+ # left. Let's try to use it for capturing more decode buckets.
1601
+ if mem_post_decode + mem_post_prompt < graph_free_mem \
1602
+ and not decode_captured_all \
1603
+ and prompt_captured_all:
1604
+ mem_post_decode, _, _ = self.warmup_graphs(
1605
+ decode_strategy,
1606
+ self.bucketing_global_state.decode_buckets, False,
1607
+ kv_caches,
1608
+ graph_free_mem - mem_post_prompt - mem_post_decode,
1609
+ mem_post_decode, decode_batch_seq)
1610
+
1611
+ self.log_graph_warmup_summary(
1612
+ self.bucketing_global_state.prompt_buckets, True,
1613
+ mem_post_prompt)
1614
+ self.log_graph_warmup_summary(
1615
+ self.bucketing_global_state.decode_buckets, False,
1616
+ mem_post_decode)
1617
+
1618
+ end_time = time.perf_counter()
1619
+ end_mem = HabanaMemoryProfiler.current_device_memory_usage()
1620
+ elapsed_time = end_time - start_time
1621
+ msg = (
1622
+ f"Warmup finished in {elapsed_time:.0f} secs, "
1623
+ f"allocated {format_bytes(end_mem - start_mem)} of device memory")
1624
+ logger.info(msg)
1625
+ self.profiler.end()
1626
+
1627
+ @property
1628
+ def vocab_size(self) -> int:
1629
+ return self.model_config.get_vocab_size()
1630
+
1631
+ @property
1632
+ def mem_margin(self) -> Optional[int]:
1633
+ return self._mem_margin
1634
+
1635
+ @mem_margin.setter
1636
+ def mem_margin(self, value):
1637
+ self._mem_margin = value
1638
+
1639
+
1640
+def _maybe_wrap_in_hpu_graph(*args, **kwargs):
1641
+ return htorch.hpu.wrap_in_hpu_graph(
1642
+ HpuModelAdapter(*args, **kwargs), disable_tensor_cache=True
1643
+ ) if htorch.utils.internal.is_lazy() else HpuModelAdapter(*args, **kwargs)
1644
+
1645
+
1646
+class HabanaProfilerCounterHelper():
1647
+
1648
+ def __init__(self):
1649
+ self.niter = 0
1650
+ self.average_real_throughput = None
1651
+ self.logged_once = False
1652
+ self.real_seq_lens = []
1653
+ self.prompt_seq_lens = []
1654
+
1655
+ def capture_seq_group_metadata_stats(self, seq_group_metadata_list):
1656
+ self.real_seq_lens = [
1657
+ len(seq_data.prompt_token_ids) + len(seq_data.output_token_ids)
1658
+ for seq_group_metadata in seq_group_metadata_list
1659
+ for seq_data in seq_group_metadata.seq_data.values()
1660
+ ]
1661
+ self.prompt_seq_lens = [
1662
+ len(seq_data.prompt_token_ids)
1663
+ for seq_group_metadata in seq_group_metadata_list
1664
+ for seq_data in seq_group_metadata.seq_data.values()
1665
+ ]
1666
+
1667
+ def get_counter_dict(self, cache_config, duration, seq_len,
1668
+ batch_size_padded, real_batch_size, is_prompt):
1669
+ throughput = batch_size_padded / (duration / 1e6)
1670
+ throughput_effective = real_batch_size / (duration / 1e6)
1671
+
1672
+ real_max_seq_len = max(self.real_seq_lens)
1673
+ real_num_tokens = sum(self.real_seq_lens)
1674
+ padded_num_tokens = batch_size_padded * seq_len
1675
+ batch_token_utilization = real_num_tokens / padded_num_tokens
1676
+ if self.average_real_throughput is None:
1677
+ self.average_real_throughput = throughput_effective
1678
+ else: # https://www.heikohoffmann.de/htmlthesis/node134.html
1679
+ self.average_real_throughput = self.average_real_throughput + 1 / (
1680
+ self.niter + 1) * (throughput_effective -
1681
+ self.average_real_throughput)
1682
+ phase = "prompt" if is_prompt else "decode"
1683
+ counters = {
1684
+ f'{phase}_bucket_batch_size': batch_size_padded,
1685
+ f'{phase}_batch_size': real_batch_size,
1686
+ f'{phase}_bucket_seq_len': seq_len,
1687
+ f'{phase}_seq_len': real_max_seq_len,
1688
+ f'{phase}_bucket_gen_throughput': throughput,
1689
+ f'{phase}_real_gen_throughput': throughput_effective,
1690
+ f'{phase}_batch_token_utilization': batch_token_utilization,
1691
+ 'average_real_throughput': self.average_real_throughput,
1692
+ 'engine_iteration': self.niter,
1693
+ }
1694
+ self.niter += 1
1695
+ if is_prompt:
1696
+ prompt_bucket_in_throughput = (seq_len * batch_size_padded) / (
1697
+ duration / 1e6)
1698
+ prompt_real_in_throughput = sum(
1699
+ self.prompt_seq_lens) / (duration / 1e6)
1700
+ counters[
1701
+ f'{phase}_bucket_in_throughput'] = prompt_bucket_in_throughput
1702
+ counters[f'{phase}_real_in_throughput'] = prompt_real_in_throughput
1703
+
1704
+ # KV cache might not be created yet (e.g. for profiling run)
1705
+ if cache_config.num_gpu_blocks is not None and \
1706
+ cache_config.num_gpu_blocks != 0:
1707
+ cache_num_blocks_used = [
1708
+ math.ceil(sl / cache_config.block_size)
1709
+ for sl in self.real_seq_lens
1710
+ ]
1711
+ cache_total_num_blocks_used = sum(cache_num_blocks_used)
1712
+ num_cache_blocks = cache_config.num_gpu_blocks
1713
+ cache_total_num_free_blocks = \
1714
+ num_cache_blocks - cache_total_num_blocks_used
1715
+ cache_computed_utilization = \
1716
+ cache_total_num_blocks_used / num_cache_blocks
1717
+ max_blocks_per_seq = math.ceil(seq_len / cache_config.block_size)
1718
+ batch_block_utilization = cache_total_num_blocks_used / (
1719
+ batch_size_padded * max_blocks_per_seq)
1720
+ counters['cache_num_blocks_used'] = cache_total_num_blocks_used
1721
+ counters['cache_num_free_blocks'] = cache_total_num_free_blocks
1722
+ counters['cache_computed_utilization'] = cache_computed_utilization
1723
+ counters[
1724
+ f'{phase}_batch_block_utilization'] = batch_block_utilization
1725
+ if not self.logged_once:
1726
+ counters['const_cache_num_blocks'] = cache_config.num_gpu_blocks
1727
+ counters[
1728
+ 'const_gpu_memory_utilization'] = \
1729
+ cache_config.gpu_memory_utilization
1730
+ counters['const_block_size'] = cache_config.block_size
1731
+ self.logged_once = True
1732
+ return counters
1733
+
1734
+
1735
+def unwrap_model(model):
1736
+ if isinstance(model, torch._dynamo.eval_frame.OptimizedModule):
1737
+ return unwrap_model(model._orig_mod)
1738
+ else:
1739
+ model = list(vars(model)['_modules'].values())[0]
1740
+ modules = list(vars(model)['_modules'].values())
1741
+ return modules
1742
+
1743
+
1744
+class HPUModelRunner(HPUModelRunnerBase[ModelInputForHPUWithSamplingMetadata]):
1745
+ """
1746
+ GPU model runner with sampling step.
1747
+ """
1748
+ _model_input_cls: Type[ModelInputForHPUWithSamplingMetadata] = (
1749
+ ModelInputForHPUWithSamplingMetadata)
1750
+
1751
+ def make_model_input_from_broadcasted_tensor_dict(
1752
+ self,
1753
+ tensor_dict: Dict[str, Any],
1754
+ ) -> ModelInputForHPUWithSamplingMetadata:
1755
+ return (
1756
+ ModelInputForHPUWithSamplingMetadata.from_broadcasted_tensor_dict(
1757
+ tensor_dict,
1758
+ attn_backend=self.attn_backend,
1759
+ ))
1760
+
1761
+ @torch.inference_mode()
1762
+ def prepare_model_input(
1763
+ self,
1764
+ seq_group_metadata_list: List[SequenceGroupMetadata],
1765
+ virtual_engine: int = 0,
1766
+ finished_requests_ids: Optional[List[str]] = None
1767
+ ) -> ModelInputForHPUWithSamplingMetadata:
1768
+ """Prepare the model input based on a given sequence group, including
1769
+ metadata for the sampling step.
1770
+ The API assumes seq_group_metadata_list is sorted by prefill -> decode.
1771
+ The result tensors and data structure also batches input in prefill
1772
+ -> decode order. For example,
1773
+ - input_tokens[:num_prefill_tokens] contains prefill tokens.
1774
+ - input_tokens[num_prefill_tokens:] contains decode tokens.
1775
+ If cuda graph is required, this API automatically pads inputs.
1776
+ """
1777
+ with self.profiler.record_event('internal', 'prepare_input_tensors'):
1778
+ assert seq_group_metadata_list is not None
1779
+ if self.profiler.enabled:
1780
+ self.profiler_counter_helper.capture_seq_group_metadata_stats(
1781
+ seq_group_metadata_list=seq_group_metadata_list)
1782
+ model_input, sampling_metadata = self.prepare_input_tensors(
1783
+ seq_group_metadata_list)
1784
+ assert model_input.attn_metadata is not None
1785
+ is_prompt = model_input.attn_metadata.is_prompt
1786
+
1787
+ return dataclasses.replace(model_input,
1788
+ sampling_metadata=sampling_metadata,
1789
+ is_prompt=is_prompt,
1790
+ virtual_engine=virtual_engine)
1791
+
1792
+ def finish_measurements(self):
1793
+ from neural_compressor.torch.quantization import finalize_calibration
1794
+ finalize_calibration(self.model.model)
1795
+
1796
+ def _check_config(self, batch_size, seq_len, is_prompt, warmup_mode):
1797
+ cfg = (batch_size, seq_len, is_prompt)
1798
+ seen = cfg in self.seen_configs
1799
+ self.seen_configs.add(cfg)
1800
+ if not seen and not warmup_mode:
1801
+ phase = 'prompt' if is_prompt else 'decode'
1802
+ logger.warning("Configuration: (%s, %s, %s) was not warmed-up!",
1803
+ phase, batch_size, seq_len)
1804
+
1805
+ def create_lora_mask(self, input_tokens: torch.Tensor, lora_ids: List[int],
1806
+ is_prompt: bool):
1807
+ '''
1808
+ This is a helper function to create the mask for lora computations.
1809
+ Lora Mask is needed to ensure we match the correct lora weights for the
1810
+ for the request.
1811
+ For Prompt phase we have
1812
+ lora_mask with shape (batch_size * seq_len, max_loras * max_rank)
1813
+ lora_logits_mask with shape (batch_size, max_loras * max_rank)
1814
+ For Decode phase we have both
1815
+ lora_mask and lora_logits_mask with shape
1816
+ (batch_size, max_loras * max_rank)
1817
+ '''
1818
+ lora_mask: torch.Tensor = None
1819
+ lora_logits_mask: torch.Tensor = None
1820
+ lora_index = 0
1821
+
1822
+ if self.lora_config:
1823
+ if is_prompt:
1824
+ lora_mask = torch.zeros(
1825
+ input_tokens.shape[0] * input_tokens.shape[1],
1826
+ (self.lora_config.max_loras) *\
1827
+ self.lora_config.max_lora_rank,
1828
+ dtype=self.lora_config.lora_dtype)
1829
+ lora_logits_mask = torch.zeros(
1830
+ input_tokens.shape[0], (self.lora_config.max_loras) *
1831
+ self.lora_config.max_lora_rank,
1832
+ dtype=self.lora_config.lora_dtype)
1833
+
1834
+ ones = torch.ones(input_tokens.shape[1],
1835
+ self.lora_config.max_lora_rank,
1836
+ dtype=self.lora_config.lora_dtype)
1837
+ logit_ones = torch.ones(1,
1838
+ self.lora_config.max_lora_rank,
1839
+ dtype=self.lora_config.lora_dtype)
1840
+
1841
+ for i in range(len(lora_ids)):
1842
+ if lora_ids[i] == 0:
1843
+ continue
1844
+ lora_index = self.lora_manager._adapter_manager.\
1845
+ lora_index_to_id.index(lora_ids[i])
1846
+ start_row = i * input_tokens.shape[1]
1847
+ end_row = start_row + input_tokens.shape[1]
1848
+ start_col = lora_index * self.lora_config.max_lora_rank
1849
+ end_col = start_col + self.lora_config.max_lora_rank
1850
+ lora_mask[start_row:end_row, start_col:end_col] = ones
1851
+ lora_logits_mask[i, start_col:end_col] = logit_ones
1852
+ lora_mask = lora_mask.to('hpu')
1853
+ lora_logits_mask = lora_logits_mask.to('hpu')
1854
+ else:
1855
+ lora_mask = torch.zeros(input_tokens.shape[0],
1856
+ (self.lora_config.max_loras) *
1857
+ self.lora_config.max_lora_rank,
1858
+ dtype=self.lora_config.lora_dtype)
1859
+ ones = torch.ones(1,
1860
+ self.lora_config.max_lora_rank,
1861
+ dtype=self.lora_config.lora_dtype)
1862
+ for i in range(len(lora_ids)):
1863
+ if lora_ids[i] == 0:
1864
+ continue
1865
+ lora_index = self.lora_manager._adapter_manager.\
1866
+ lora_index_to_id.index(lora_ids[i])
1867
+ start_pos = lora_index * self.lora_config.max_lora_rank
1868
+ end_pos = start_pos + self.lora_config.max_lora_rank
1869
+ lora_mask[i, start_pos:end_pos] = ones
1870
+ lora_mask = lora_mask.to('hpu')
1871
+ lora_logits_mask = lora_mask
1872
+
1873
+ return lora_mask, lora_logits_mask
1874
+
1875
+ @torch.inference_mode()
1876
+ def execute_model(
1877
+ self,
1878
+ model_input: ModelInputForHPUWithSamplingMetadata,
1879
+ kv_caches: List[torch.Tensor],
1880
+ intermediate_tensors: Optional[IntermediateTensors] = None,
1881
+ num_steps: int = 1,
1882
+ warmup_mode=False,
1883
+ ) -> Optional[Union[List[SamplerOutput], IntermediateTensors]]:
1884
+ if num_steps > 1:
1885
+ raise ValueError(
1886
+ "num_steps > 1 is not supported in HPUModelRunner")
1887
+
1888
+ if self.lora_config:
1889
+ assert model_input.lora_requests is not None
1890
+ assert model_input.lora_mapping is not None
1891
+ self.set_active_loras(model_input.lora_requests,
1892
+ model_input.lora_mapping)
1893
+ input_tokens = model_input.input_tokens
1894
+ input_positions = model_input.input_positions
1895
+ attn_metadata = model_input.attn_metadata
1896
+ sampling_metadata = model_input.sampling_metadata
1897
+ real_batch_size = model_input.real_batch_size
1898
+ batch_size_padded = model_input.batch_size_padded
1899
+ assert input_tokens is not None
1900
+ assert input_positions is not None
1901
+ assert sampling_metadata is not None
1902
+ assert attn_metadata is not None
1903
+ is_prompt = attn_metadata.is_prompt
1904
+ assert is_prompt is not None
1905
+ batch_size = input_tokens.size(0)
1906
+ seq_len = self._seq_len(attn_metadata)
1907
+ use_graphs = self._use_graphs(batch_size, seq_len, is_prompt)
1908
+ self._check_config(batch_size, seq_len, is_prompt, warmup_mode)
1909
+
1910
+ lora_mask: torch.Tensor = None
1911
+ lora_logits_mask: torch.Tensor = None
1912
+ if self.lora_config:
1913
+ assert model_input.lora_ids is not None
1914
+ lora_mask, lora_logits_mask = self.create_lora_mask(
1915
+ input_tokens, model_input.lora_ids, attn_metadata.is_prompt)
1916
+
1917
+ execute_model_kwargs = {
1918
+ "input_ids": input_tokens,
1919
+ "positions": input_positions,
1920
+ "kv_caches": kv_caches,
1921
+ "attn_metadata": self.trim_attn_metadata(attn_metadata),
1922
+ "intermediate_tensors": intermediate_tensors,
1923
+ "lora_mask": lora_mask,
1924
+ **(model_input.multi_modal_kwargs or {}),
1925
+ }
1926
+ if htorch.utils.internal.is_lazy():
1927
+ execute_model_kwargs.update({"bypass_hpu_graphs": not use_graphs})
1928
+
1929
+ htorch.core.mark_step()
1930
+ if self.is_driver_worker:
1931
+ model_event_name = ("model_"
1932
+ f"{'prompt' if is_prompt else 'decode'}_"
1933
+ f"bs{batch_size}_"
1934
+ f"seq{seq_len}_"
1935
+ f"graphs{'T' if use_graphs else 'F'}")
1936
+ else:
1937
+ model_event_name = 'model_executable'
1938
+ with self.profiler.record_event('internal', model_event_name):
1939
+ hidden_states = self.model.forward(
1940
+ **execute_model_kwargs,
1941
+ selected_token_indices=sampling_metadata.selected_token_indices
1942
+ )
1943
+
1944
+ if self.lora_config:
1945
+ LoraMask.setLoraMask(
1946
+ lora_logits_mask.index_select(
1947
+ 0, sampling_metadata.selected_token_indices))
1948
+
1949
+ # Compute the logits.
1950
+ with self.profiler.record_event(
1951
+ 'internal', ('compute_logits_'
1952
+ f'{"prompt" if is_prompt else "decode"}_bs'
1953
+ f'{batch_size}_'
1954
+ f'seq{seq_len}')):
1955
+ sampling_metadata.selected_token_indices = None
1956
+ logits = self.model.compute_logits(hidden_states,
1957
+ sampling_metadata)
1958
+ htorch.core.mark_step()
1959
+ # Only perform sampling in the driver worker.
1960
+ if not self.is_driver_worker:
1961
+ return []
1962
+
1963
+ if model_input.async_callback is not None:
1964
+ model_input.async_callback()
1965
+
1966
+ # Sample the next token.
1967
+ with self.profiler.record_event(
1968
+ 'internal', ('sample_'
1969
+ f'{"prompt" if is_prompt else "decode"}_'
1970
+ f'bs{batch_size}_'
1971
+ f'seq{seq_len}')):
1972
+ output = self.model.sample(
1973
+ logits=logits,
1974
+ sampling_metadata=sampling_metadata,
1975
+ )
1976
+ output.outputs = output.outputs[:real_batch_size]
1977
+ htorch.core.mark_step()
1978
+
1979
+ if self.is_driver_worker and self.profiler.enabled:
1980
+ # Stop recording 'execute_model' event
1981
+ self.profiler.end()
1982
+ event_end = self.profiler.get_timestamp_us()
1983
+ counters = self.profiler_counter_helper.get_counter_dict(
1984
+ cache_config=self.cache_config,
1985
+ duration=event_end - self.event_start,
1986
+ seq_len=seq_len,
1987
+ batch_size_padded=batch_size_padded,
1988
+ real_batch_size=real_batch_size,
1989
+ is_prompt=is_prompt)
1990
+ self.profiler.record_counter(self.event_start, counters)
1991
+ return [output]
1992
+
1993
+ def shutdown_inc(self):
1994
+ can_finalize_inc = False
1995
+ from contextlib import suppress
1996
+ with suppress(AttributeError):
1997
+ can_finalize_inc = (self.model_config.quantization == 'inc') and \
1998
+ (self.model.model is not None) and \
1999
+ self.inc_initialized_successfully and \
2000
+ not getattr(self, "_is_inc_finalized", False)
2001
+ if can_finalize_inc:
2002
+ from neural_compressor.torch.quantization import (
2003
+ finalize_calibration)
2004
+ finalize_calibration(self.model.model)
2005
+ self._is_inc_finalized = True
2006
+
2007
+ def __del__(self):
2008
+ self.shutdown_inc()