A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
New CI template on AWS stack (#5110)
Signed-off-by: kevin <kevin@anyscale.com>
Kevin H. Luu committed
Jun 3, 2024 at 16:16 UTC
4f0d17c05cdb220f2f45a20e956f766dec29acbc
1 file changed
+59
.buildkite/test-template-aws.j2
new
+59
@@ -0,0 +1,59 @@
1
+{% set docker_image = "public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT" %}
2
+{% set default_working_dir = "/vllm-workspace/tests" %}
3
+
4
+steps:
5
+ - label: ":docker: build image"
6
+ agents:
7
+ queue: cpu_queue
8
+ commands:
9
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
10
+ - "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
11
+ - "docker push {{ docker_image }}"
12
+ env:
13
+ DOCKER_BUILDKIT: "1"
14
+ retry:
15
+ automatic:
16
+ - exit_status: -1 # Agent was lost
17
+ limit: 5
18
+ - exit_status: -10 # Agent was lost
19
+ limit: 5
20
+ - wait
21
+
22
+ {% for step in steps %}
23
+ - label: "{{ step.label }}"
24
+ agents:
25
+ {% if step.no_gpu %}
26
+ queue: cpu_queue
27
+ {% elif step.num_gpus == 2 or step.num_gpus == 4 %}
28
+ queue: gpu_4_queue
29
+ {% else %}
30
+ queue: gpu_1_queue
31
+ {% endif %}
32
+ soft_fail: true
33
+ {% if step.parallelism %}
34
+ parallelism: {{ step.parallelism }}
35
+ {% endif %}
36
+ retry:
37
+ automatic:
38
+ - exit_status: -1 # Agent was lost
39
+ limit: 5
40
+ - exit_status: -10 # Agent was lost
41
+ limit: 5
42
+ plugins:
43
+ - docker#v5.2.0:
44
+ image: {{ docker_image }}
45
+ always-pull: true
46
+ propagate-environment: true
47
+ {% if not step.no_gpu %}
48
+ gpus: all
49
+ {% endif %}
50
+ command: ["bash", "-c", "cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}"]
51
+ environment:
52
+ - VLLM_USAGE_SOURCE=ci-test
53
+ - HF_TOKEN
54
+ {% if step.label == "Speculative decoding tests" %}
55
+ - VLLM_ATTENTION_BACKEND=XFORMERS
56
+ {% endif %}
57
+ volumes:
58
+ - /dev/shm:/dev/shm
59
+ {% endfor %}