@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1fa020c539
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1][BugFix] Fix Generator construction in greedy + seed case (#10097)

Signed-off-by: Nick Hill <nhill@redhat.com>

Nick Hill committed Nov 7, 2024 at 05:06 UTC 1fa020c539485e398d10ca9be376c1d0d87ae19b
1 file changed +3 -2
vllm/v1/worker/gpu_model_runner.py
+3 -2
@@ -146,7 +146,7 @@ class GPUModelRunner:
146 for req_data in scheduler_output.scheduled_new_reqs:
147 req_id = req_data.req_id
148 sampling_params = req_data.sampling_params
149 - if sampling_params.seed is not None:
149 + if sampling_params.sampling_type == SamplingType.RANDOM_SEED:
150 generator = torch.Generator(device=self.device)
151 generator.manual_seed(sampling_params.seed)
152 else:
@@ -382,7 +382,8 @@ class GPUModelRunner:
382 # Rewind the generator state as if the token was not sampled.
383 generator = self.input_batch.generators.get(i)
384 if generator is not None:
385 - generator.set_offset(generator.get_offset() - 1)
385 + # This relies on cuda-specific torch-internal impl details
386 + generator.set_offset(generator.get_offset() - 4)
387
388 if sampler_output.logprob_token_ids is None:
389 logprob_token_ids = None