A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Comment out unused code in sampler (#7023)
Peng Guanwen committed
Aug 2, 2024 at 15:58 UTC
db35186391a2abfc6c91d703527dac20d2488107
1 file changed
+31
-27
vllm/model_executor/sampling_metadata.py
+31
-27
@@ -13,6 +13,8 @@ from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
13
14
_SAMPLING_EPS = 1e-5
15
_SEED_0_REPLACEMENT = 3403598558
16
+# Some triton sampler related code is guarded before it is ready.
17
+_USE_TRITON_SAMPLER = False
18
19
20
@dataclass
@@ -347,14 +349,16 @@ class SamplingTensors:
349
repetition_penalties: List[float] = []
350
sampling_seeds: List[int] = []
351
sample_indices: List[int] = []
350
- prompt_best_of: List[int] = []
352
do_penalties = False
353
do_top_p_top_k = False
354
do_min_p = False
355
355
- # We need one base seed per Triton slice.
356
- seeds_to_generate = (extra_seeds_to_generate +
357
- get_num_triton_sampler_splits(vocab_size))
356
+ if _USE_TRITON_SAMPLER:
357
+ prompt_best_of: List[int] = []
358
+
359
+ # We need one base seed per Triton slice.
360
+ seeds_to_generate = (extra_seeds_to_generate +
361
+ get_num_triton_sampler_splits(vocab_size))
362
363
assert sampling_metadata.seq_groups is not None
364
for seq_group in sampling_metadata.seq_groups:
@@ -366,9 +370,6 @@ class SamplingTensors:
370
r = sampling_params.repetition_penalty
371
top_p = sampling_params.top_p
372
min_p = sampling_params.min_p
369
- seed = sampling_params.seed
370
-
371
- is_greedy = sampling_params.sampling_type == SamplingType.GREEDY
373
374
# k should not be greater than the vocab size.
375
top_k = min(sampling_params.top_k, vocab_size)
@@ -389,8 +390,7 @@ class SamplingTensors:
390
do_penalties = True
391
392
is_prompt = seq_group.is_prompt
392
- if (seq_group.is_prompt
393
- and sampling_params.prompt_logprobs is not None):
393
+ if (is_prompt and sampling_params.prompt_logprobs is not None):
394
# For tokens in the prompt that we only need to get
395
# their logprobs
396
query_len = seq_group.query_len
@@ -415,23 +415,27 @@ class SamplingTensors:
415
frequency_penalties += [f] * len(seq_ids)
416
repetition_penalties += [r] * len(seq_ids)
417
418
- if is_prompt:
419
- prompt_best_of.append(sampling_params.best_of)
420
- query_len = seq_group.query_len
421
- assert query_len is not None
422
-
423
- for seq_id in seq_ids:
424
- seq_data = seq_group.seq_data[seq_id]
425
- extra_entropy = extra_entropy or ()
426
- seq_seeds = cls._get_sequence_seeds(
427
- seed,
428
- seq_data.get_len(),
429
- *extra_entropy,
430
- seq_id,
431
- seeds_to_generate=seeds_to_generate,
432
- is_greedy=is_greedy)
433
- sampling_seeds.append(seq_seeds)
434
- sample_indices.extend(seq_group.sample_indices)
418
+ if _USE_TRITON_SAMPLER:
419
+ if is_prompt:
420
+ prompt_best_of.append(sampling_params.best_of)
421
+ query_len = seq_group.query_len
422
+ assert query_len is not None
423
+
424
+ seed = sampling_params.seed
425
+ is_greedy = sampling_params.sampling_type == SamplingType.GREEDY
426
+
427
+ for seq_id in seq_ids:
428
+ seq_data = seq_group.seq_data[seq_id]
429
+ extra_entropy = extra_entropy or ()
430
+ seq_seeds = cls._get_sequence_seeds(
431
+ seed,
432
+ seq_data.get_len(),
433
+ *extra_entropy,
434
+ seq_id,
435
+ seeds_to_generate=seeds_to_generate,
436
+ is_greedy=is_greedy)
437
+ sampling_seeds.append(seq_seeds)
438
+ sample_indices.extend(seq_group.sample_indices)
439
440
if do_penalties:
441
for seq_group in sampling_metadata.seq_groups:
@@ -549,7 +553,7 @@ class SamplingTensors:
553
device="cpu",
554
dtype=torch.long,
555
pin_memory=pin_memory,
552
- ).T.contiguous()
556
+ ).t().contiguous()
557
558
# Because the memory is pinned, we can do non-blocking
559
# transfer to device.