@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4f0d17c05c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

New CI template on AWS stack (#5110)

Signed-off-by: kevin <kevin@anyscale.com>

Kevin H. Luu committed Jun 3, 2024 at 16:16 UTC 4f0d17c05cdb220f2f45a20e956f766dec29acbc
1 file changed +59
.buildkite/test-template-aws.j2 new
+59
@@ -0,0 +1,59 @@
1 +{% set docker_image = "public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" %}
2 +{% set default_working_dir = "/vllm-workspace/tests" %}
3 +
4 +steps:
5 + - label: ":docker: build image"
6 + agents:
7 + queue: cpu_queue
8 + commands:
9 + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
10 + - "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
11 + - "docker push {{ docker_image }}"
12 + env:
13 + DOCKER_BUILDKIT: "1"
14 + retry:
15 + automatic:
16 + - exit_status: -1 # Agent was lost
17 + limit: 5
18 + - exit_status: -10 # Agent was lost
19 + limit: 5
20 + - wait
21 +
22 + {% for step in steps %}
23 + - label: "{{ step.label }}"
24 + agents:
25 + {% if step.no_gpu %}
26 + queue: cpu_queue
27 + {% elif step.num_gpus == 2 or step.num_gpus == 4 %}
28 + queue: gpu_4_queue
29 + {% else %}
30 + queue: gpu_1_queue
31 + {% endif %}
32 + soft_fail: true
33 + {% if step.parallelism %}
34 + parallelism: {{ step.parallelism }}
35 + {% endif %}
36 + retry:
37 + automatic:
38 + - exit_status: -1 # Agent was lost
39 + limit: 5
40 + - exit_status: -10 # Agent was lost
41 + limit: 5
42 + plugins:
43 + - docker#v5.2.0:
44 + image: {{ docker_image }}
45 + always-pull: true
46 + propagate-environment: true
47 + {% if not step.no_gpu %}
48 + gpus: all
49 + {% endif %}
50 + command: ["bash", "-c", "cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}"]
51 + environment:
52 + - VLLM_USAGE_SOURCE=ci-test
53 + - HF_TOKEN
54 + {% if step.label == "Speculative decoding tests" %}
55 + - VLLM_ATTENTION_BACKEND=XFORMERS
56 + {% endif %}
57 + volumes:
58 + - /dev/shm:/dev/shm
59 + {% endfor %}