A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1][BugFix] Fix Generator construction in greedy + seed case (#10097)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Nov 7, 2024 at 05:06 UTC
1fa020c539485e398d10ca9be376c1d0d87ae19b
1 file changed
+3
-2
vllm/v1/worker/gpu_model_runner.py
+3
-2
@@ -146,7 +146,7 @@ class GPUModelRunner:
146
for req_data in scheduler_output.scheduled_new_reqs:
147
req_id = req_data.req_id
148
sampling_params = req_data.sampling_params
149
- if sampling_params.seed is not None:
149
+ if sampling_params.sampling_type == SamplingType.RANDOM_SEED:
150
generator = torch.Generator(device=self.device)
151
generator.manual_seed(sampling_params.seed)
152
else:
@@ -382,7 +382,8 @@ class GPUModelRunner:
382
# Rewind the generator state as if the token was not sampled.
383
generator = self.input_batch.generators.get(i)
384
if generator is not None:
385
- generator.set_offset(generator.get_offset() - 1)
385
+ # This relies on cuda-specific torch-internal impl details
386
+ generator.set_offset(generator.get_offset() - 4)
387
388
if sampler_output.logprob_token_ids is None:
389
logprob_token_ids = None