Introduce new CI to validate new distributions entries (#12314)
* Introduce new CI to validate new distributions entries
Blue committed
Nov 26, 2024 at 13:33 UTC
ef61f61274fd792cea94ce27af5f62adbdaf9edc
4 files changed
+478
.github/workflows/modern-distributions.yml
new
+34
@@ -0,0 +1,34 @@
1
+name: Validate tar based distributions
2
+
3
+on:
4
+ pull_request:
5
+ paths: ['distributions/**']
6
+
7
+permissions:
8
+ pull-requests: write
9
+
10
+jobs:
11
+ check:
12
+ name: Validate tar based distributions changes
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - name: Checkout repo
16
+ uses: actions/checkout@v4
17
+ with:
18
+ fetch-depth: 0
19
+
20
+
21
+ - name: Install pip packages
22
+ run: pip install -r distributions/requirements.txt
23
+ shell: bash
24
+
25
+ - name: Run validation
26
+ run: |
27
+ python distributions/validate-modern.py \
28
+ --repo-path . \
29
+ --compare-with-branch 'origin/${{ github.base_ref }}' \
30
+ --manifest distributions/DistributionInfo.json \
31
+ --github-token '${{ secrets.GITHUB_TOKEN }}' \
32
+ --github-pr '${{ github.event.pull_request.number }}' \
33
+ --github-commit '${{ github.event.pull_request.merge_commit_sha }}'
34
+ shell: bash
distributions/DistributionInfo.json
+1
@@ -1,4 +1,5 @@
1
{
2
+ "ModernDistributions": {},
3
"Distributions": [
4
{
5
"Name": "Ubuntu",
distributions/requirements.txt
new
+4
@@ -0,0 +1,4 @@
1
+python-magic==0.4.27
2
+click==8.1.3
3
+GitPython==3.1.29
4
+PyGithub==2.5.0
\ No newline at end of file
distributions/validate-modern.py
new
+439
@@ -0,0 +1,439 @@
1
+import click
2
+import json
3
+import requests
4
+import tempfile
5
+import hashlib
6
+import tarfile
7
+import configparser
8
+import magic
9
+import os.path
10
+import git
11
+import re
12
+from github import Github
13
+
14
+
15
+USR_LIB_WSL = '/usr/lib/wsl'
16
+
17
+MAGIC = magic.Magic()
18
+X64_ELF_MAGIC = re.compile('^ELF 64-bit.* x86-64, version 1')
19
+ARM64_ELF_MAGIC = re.compile('^ELF 64-bit.* ARM aarch64, version 1')
20
+
21
+DISCOURAGED_SYSTEM_UNITS = ['systemd-resolved.service',
22
+ 'systemd-networkd.service',
23
+ 'systemd-tmpfiles-setup.service',
24
+ 'systemd-tmpfiles-clean.service',
25
+ 'systemd-tmpfiles-setup-dev-early.service',
26
+ 'systemd-tmpfiles-setup-dev.service',
27
+ 'tmp.mount',
28
+ 'NetworkManager.service',
29
+ 'networking.service']
30
+
31
+errors = []
32
+warnings = []
33
+
34
+
35
+@click.command()
36
+@click.option('--manifest', default=None)
37
+@click.option('--tar', default=None)
38
+@click.option('--compare-with-branch')
39
+@click.option('--repo-path', '..')
40
+@click.option('--arm64', is_flag=True)
41
+@click.option('--github-token', default=None)
42
+@click.option('--github-pr', default=None, type=int)
43
+@click.option('--github-commit', default=None)
44
+@click.option('--debug', is_flag=True)
45
+def main(manifest: str, tar: str, compare_with_branch: str, repo_path: str, arm64: bool, github_token: str, github_pr: str, github_commit: str, debug: bool):
46
+ try:
47
+ if tar is not None:
48
+ with open(tar, 'rb') as fd:
49
+ read_tar(tar, '<none>', fd, ARM64_ELF_MAGIC if arm64 else X64_ELF_MAGIC)
50
+ else:
51
+ if manifest is None:
52
+ raise RuntimeError('Either --tar or --manifest is required')
53
+
54
+ with open(manifest) as fd:
55
+ manifest_content = json.loads(fd.read())
56
+
57
+ baseline_manifest = None
58
+ if compare_with_branch is not None:
59
+ repo = git.Repo(repo_path)
60
+ baseline_json = repo.commit(compare_with_branch).tree / 'distributions/DistributionInfo.json'
61
+ baseline_manifest = json.load(baseline_json.data_stream).get('ModernDistributions', {})
62
+
63
+ for flavor, versions in manifest_content["ModernDistributions"].items():
64
+ baseline_flavor = baseline_manifest.get(flavor, None) if baseline_manifest else None
65
+
66
+ for e in versions:
67
+ name = e.get('Name', None)
68
+
69
+ if name is None:
70
+ error(flavor, None, 'Found nameless distribution')
71
+ continue
72
+
73
+ if baseline_flavor is not None:
74
+ baseline_version = next((entry for entry in baseline_flavor if entry['Name'] == name), None)
75
+ if baseline_version is None:
76
+ click.secho(f'Found new entry for flavor "{flavor}": {name}', fg='green', bold=True)
77
+ elif baseline_version != e:
78
+ click.secho(f'Found changed entry for flavor "{flavor}": {name}', fg='green', bold=True)
79
+ else:
80
+ click.secho(f'Distribution entry "{flavor}/{name}" is unchanged, skipping')
81
+ continue
82
+
83
+ click.secho(f'Reading information for distribution: {e["Name"]}', bold=True)
84
+ if 'FriendlyName' not in e:
85
+ error(flavor, name, 'Manifest entry is missing a "FriendlyName" entry')
86
+
87
+ if not name.startswith(flavor):
88
+ error(flavor, name, f'Name should start with "{flavor}"')
89
+
90
+ url_found = False
91
+
92
+ if 'Amd64Url' in e:
93
+ read_url(flavor, name, e['Amd64Url'], X64_ELF_MAGIC)
94
+ url_found = True
95
+
96
+ if 'Arm64Url' in e:
97
+ read_url(flavor, name, e['Arm64Url'], ARM64_ELF_MAGIC)
98
+ url_found = True
99
+
100
+ if not url_found:
101
+ error(flavor, name, 'No URL found')
102
+
103
+ expectedKeys = ['Name', 'FriendlyName', 'Default', 'Amd64Url', 'Arm64Url']
104
+ for key in e.keys():
105
+ if key not in expectedKeys:
106
+ error(flavor, name, 'Unexpected key: "{key}"')
107
+
108
+
109
+ default_entries = sum(1 for e in versions if e.get('Default', False))
110
+ if default_entries != 1:
111
+ error(flavor, None, 'Found no default distribution' if default_entries == 0 else 'Found multiple default distributions')
112
+
113
+ if github_pr is not None:
114
+ assert github_token is not None and github_commit is not None and manifest is not None
115
+
116
+ report_status_on_pr(github_pr, github_token, github_commit, manifest)
117
+
118
+ except:
119
+ if debug:
120
+ import traceback
121
+ traceback.print_exc()
122
+ import pdb
123
+ pdb.post_mortem()
124
+ else:
125
+ raise
126
+
127
+def report_status_on_pr(pr: int, github_token: str, github_commit: str, manifest: str):
128
+ github = Github(github_token)
129
+ repo = github.get_repo('microsoft/WSL')
130
+
131
+ def format_list(entries: list) -> str:
132
+ output = '\n'
133
+
134
+ for e in entries:
135
+ output += f'\n* {e}'
136
+
137
+ return output + '\n'
138
+
139
+ body = 'Thank you for your contribution to WSL.\n'
140
+ if errors:
141
+ body += f'**The following fatal errors have been found in this pull request:** {format_list(errors)}\n'
142
+ else:
143
+ body += 'No fatal errors have been found.\n'
144
+
145
+ if warnings:
146
+ body += f'**The following suggestions have been found in this pull request:** {format_list(warnings)}\n'
147
+ else:
148
+ body += 'No suggestions have been found.\n'
149
+
150
+ repo.get_pull(pr).create_review(body=body, commit=repo.get_commit(github_commit))
151
+
152
+
153
+def read_config_keys(config: configparser.ConfigParser) -> dict:
154
+ keys = {}
155
+
156
+ for section in config.sections():
157
+ for key in config[section].keys():
158
+ keys[f'{section}.{key}'] = config[section][key]
159
+
160
+ return keys
161
+
162
+def read_passwd(flavor: str, name: str, default_uid: int, fd):
163
+ def read_passwd_line(line: str):
164
+ fields = line.split(':')
165
+
166
+ if len(fields) != 7:
167
+ error(flavor, name, f'Invalid passwd entry: {line}')
168
+ return None, None
169
+ try:
170
+ uid = int(fields[2])
171
+ except ValueError:
172
+ error(flavor, name, f'Invalid passwd entry: {line}')
173
+ return None, None
174
+
175
+ return uid, fields
176
+
177
+ entries = {}
178
+
179
+ for line in fd.readlines():
180
+ uid, fields = read_passwd_line(line.decode())
181
+
182
+ if uid in entries:
183
+ error(flavor, name, f'found duplicated uid in /etc/passw: {uid}')
184
+ else:
185
+ entries[uid] = fields
186
+
187
+ if 0 not in entries:
188
+ error(flavor, name, f'No root (uid=0) found in /etc/passwd')
189
+ elif entries[0][0] != 'root':
190
+ error(flavor, name, f'/etc/passwd has a uid=0, but it is not root: {entries[0][0]}')
191
+
192
+ if default_uid is not None and default_uid in entries:
193
+ warning(flavor, name, f'/etc/passwd already has an entry for default uid: {entries[default_uid]}')
194
+
195
+# This logic isn't perfect at listing all boot units, but parsing all of systemd configuration would be too complex.
196
+def read_systemd_enabled_units(flavor: str, name: str, tar) -> dict:
197
+ config_dirs = ['/usr/local/lib/systemd/system', '/usr/lib/systemd/system', '/etc/systemd/system']
198
+
199
+ all_files = tar.getnames()
200
+
201
+ def link_target(unit_path: str):
202
+ try:
203
+ info = tar.getmember(unit_path)
204
+ except KeyError:
205
+ info = tar.getmember('.' + unit_path)
206
+
207
+ if not info.issym():
208
+ return unit_path
209
+ else:
210
+ return info.linkpath
211
+
212
+ def list_directory(path: str):
213
+ files = []
214
+ for e in all_files:
215
+ if e.startswith(path):
216
+ files.append(e[len(path) + 1:])
217
+ elif e.startswith('.' + path):
218
+ files.append(e[len(path) + 2:])
219
+
220
+ return files
221
+
222
+ units = {}
223
+ for config_dir in config_dirs:
224
+ targets = [e for e in list_directory(config_dir) if e.endswith('.target.wants')]
225
+
226
+ for target in targets:
227
+ for e in list_directory(f'{config_dir}/{target}'):
228
+ fullpath = f'{config_dir}/{target}/{e}'
229
+ unit_target = link_target(fullpath)
230
+
231
+ if unit_target != '/dev/null':
232
+ units[e] = fullpath
233
+
234
+ return units
235
+
236
+
237
+def get_tar_file(tar, path: str, follow_symlink=False):
238
+
239
+ # Tar members can be formated as /{path}, {path}, or ./{path}
240
+ if path.startswith('/'):
241
+ paths = [path, '.' + path, path[1:]]
242
+ elif path.startswith('./'):
243
+ paths = [path, path[1:], path[2:]]
244
+ else:
245
+ paths = [path, './' + path, '/' + path]
246
+
247
+ def follow_if_symlink(info, path: str):
248
+ if follow_symlink and info.issym():
249
+ if info.linkpath.startswith('/'):
250
+ return get_tar_file(tar, info.linkpath, follow_symlink=True)
251
+ else:
252
+ return get_tar_file(tar, f'{os.path.dirname(path)}/{info.linkpath}', follow_symlink=True)
253
+ else:
254
+ return info, path
255
+
256
+ # First try accessing the file directly
257
+ for e in paths:
258
+ try:
259
+ return follow_if_symlink(tar.getmember(e), e)
260
+ except KeyError:
261
+ continue
262
+
263
+ if not follow_symlink:
264
+ return None, None
265
+
266
+ # Then look for symlinks
267
+ # The path might be covered by a symlink, check if parent exists and is a symlink
268
+ parent_path = os.path.dirname(path)
269
+ if parent_path != path:
270
+ try:
271
+ parent_info, real_parent_path = get_tar_file(tar, parent_path, follow_symlink=True)
272
+ if real_parent_path != parent_path:
273
+ return get_tar_file(tar, f'{real_parent_path}/{os.path.basename(path)}', follow_symlink=True)
274
+ except KeyError:
275
+ pass
276
+
277
+ return None, None
278
+
279
+def read_tar(flavor: str, name: str, file, elf_magic: str):
280
+ with tarfile.open(fileobj=file) as tar:
281
+
282
+ def validate_mode(path: str, mode, uid, gid, max_size = None, optional = False, follow_symlink = False, magic = None, parse_method = None):
283
+ info, real_path = get_tar_file(tar, path, follow_symlink)
284
+
285
+ if info is None:
286
+ if not optional:
287
+ error(flavor, name, f'File "{path}" not found in tar')
288
+ return False
289
+
290
+ permissions = oct(info.mode)
291
+ if permissions not in mode:
292
+ warning(flavor, name, f'file: "{path}" has unexpected mode: {permissions} (expected: {mode})')
293
+
294
+ if info.uid != uid:
295
+ warning(flavor, name, f'file: "{path}" has unexpected uid: {info.uid} (expected: {uid})')
296
+
297
+ if gid is not None and info.gid != gid:
298
+ warning(flavor, name, f'file: "{path}" has unexpected gid: {info.gid} (expected: {gid})')
299
+
300
+ if max_size is not None and info.size > max_size:
301
+ error(flavor, name, f'file: "{path}" is too big (info.size), max: {max_size}')
302
+
303
+ if magic is not None or parse_method is not None:
304
+ content = tar.extractfile(real_path)
305
+
306
+ if parse_method is not None:
307
+ parse_method(content)
308
+
309
+ if magic is not None:
310
+ content.seek(0)
311
+ buffer = content.read(256)
312
+ file_magic = MAGIC.from_buffer(buffer)
313
+ if not magic.match(file_magic):
314
+ error(flavor, name, f'file: "{path}" has unexpected magic type: {file_magic} (expected: {magic})')
315
+
316
+ return True
317
+
318
+ def validate_config(path: str, valid_keys: list):
319
+ _, path = get_tar_file(tar, path, follow_symlink=True)
320
+ if path is None:
321
+ error(flavor, name, f'File "{file}" not found in tar')
322
+ return None
323
+
324
+ content = tar.extractfile(path)
325
+ config = configparser.ConfigParser()
326
+ config.read_string(content.read().decode())
327
+
328
+ keys = read_config_keys(config)
329
+
330
+ unexpected_keys = [e for e in keys if e not in valid_keys]
331
+ if unexpected_keys:
332
+ error(flavor, name, f'Found unexpected_keys in "{path}": {unexpected_keys}')
333
+ else:
334
+ click.secho(f'Found valid keys in "{path}": {list(keys.keys())}')
335
+
336
+ return keys
337
+
338
+ defaultUid = None
339
+ if validate_mode('/etc/wsl-distribution.conf', [oct(0o664), oct(0o644)], 0, 0):
340
+ config = validate_config('/etc/wsl-distribution.conf', ['oobe.command', 'oobe.defaultuid', 'shortcut.icon', 'oobe.defaultname', 'windowsterminal.profileTemplate'])
341
+
342
+ if oobe_command := config.get('oobe.command', None):
343
+ validate_mode(oobe_command, [oct(0o775), oct(0o755)], 0, 0)
344
+
345
+ if not oobe_command.startswith(USR_LIB_WSL):
346
+ warning(flavor, name, f'value for oobe.command is not under {USR_LIB_WSL}: "{oobe_command}"')
347
+
348
+ if defaultUid := config.get('oobe.defaultuid', None):
349
+ if defaultUid != '1000':
350
+ warning(flavor, name, f'Default UID is not 1000. Found: {defaultUid}')
351
+
352
+ defaultUid = int(defaultUid)
353
+
354
+ if shortcut_icon := config.get('shortcut.icon', None):
355
+ validate_mode(shortcut_icon, [oct(0o664), oct(0o644)], 0, 0, 1024 * 1024)
356
+
357
+ if not shortcut_icon.startswith(USR_LIB_WSL):
358
+ warning(flavor, name, f'value for shortcut.icon is not under {USR_LIB_WSL}: "{shortcut_icon}"')
359
+
360
+ if terminal_profile := config.get('windowsterminal.profileTemplate', None):
361
+ validate_mode(terminal_profile, [oct(0o660), oct(0o640)], 0, 0, 1024 * 1024)
362
+
363
+ if not terminal_profile.startswith(USR_LIB_WSL):
364
+ warning(flavor, name, f'value for windowsterminal.profileTemplate is not under {USR_LIB_WSL}: "{terminal_profile}"')
365
+
366
+ if validate_mode('/etc/wsl.conf', [oct(0o664), oct(0o644)], 0, 0, optional=True):
367
+ config = validate_config('/etc/wsl.conf', ['boot.systemd'])
368
+ if config.get('boot.systemd', False):
369
+ validate_mode('/sbin/init', [oct(0o775), oct(0o755)], 0, 0, magic=elf_magic, follow_symlink=True)
370
+
371
+ validate_mode('/etc/passwd', [oct(0o664), oct(0o644)], 0, 0, parse_method = lambda fd: read_passwd(flavor, name, defaultUid, fd))
372
+ validate_mode('/etc/shadow', [oct(0o640), oct(0o600)], 0, None)
373
+ validate_mode('/bin/bash', [oct(0o755), oct(0o775)], 0, 0, magic=elf_magic, follow_symlink=True)
374
+ validate_mode('/bin/sh', [oct(0o755), oct(0o775)], 0, 0, magic=elf_magic, follow_symlink=True)
375
+
376
+ enabled_systemd_units = read_systemd_enabled_units(flavor, name, tar)
377
+ for unit, path in enabled_systemd_units.items():
378
+ if unit in DISCOURAGED_SYSTEM_UNITS:
379
+ warning(flavor, name, f'Found discouraged system unit: {path}')
380
+
381
+def read_url(flavor: str, name: str, url: dict, elf_magic):
382
+ hash = hashlib.sha256()
383
+
384
+ if url['Url'].startswith('file://'):
385
+ with open(url['Url'].replace('file:///', '').replace('file://', ''), 'rb') as fd:
386
+ while True:
387
+ e = fd.read(4096 * 4096 * 10)
388
+ if not e:
389
+ break
390
+
391
+ hash.update(e)
392
+
393
+ fd.seek(0, 0)
394
+ read_tar(flavor, name, fd, elf_magic)
395
+ else:
396
+ with requests.get(url['Url'], stream=True) as response:
397
+ response.raise_for_status()
398
+
399
+ with tempfile.NamedTemporaryFile() as file:
400
+ for e in response.iter_content(chunk_size=4096 * 4096):
401
+ file.write(e)
402
+ hash.update(e)
403
+
404
+ file.seek(0, 0)
405
+ read_tar(flavor, name, file, elf_magic)
406
+
407
+
408
+ expected_sha = url.get('Sha256', None)
409
+ if expected_sha is None:
410
+ error(flavor, name, 'URL is missing "Sha256"')
411
+ else:
412
+ if expected_sha.startswith('0x'):
413
+ expected_sha = expected_sha[2:]
414
+
415
+ sha = hash.digest()
416
+ if bytes.fromhex(expected_sha) != sha:
417
+ error(flavor, name, f'URL {url["Url"]} Sha256 does not match. Expected: {expected_sha}, actual: {hash.hexdigest()}')
418
+ else:
419
+ click.secho(f'Hash for {url["Url"]} matches ({expected_sha})', fg='green')
420
+
421
+
422
+
423
+def error(flavor: str, distribution: str, message: str):
424
+ global errors
425
+
426
+ message = f'{flavor}/{distribution}: {message}'
427
+ click.secho(f'Error: {message}', fg='red')
428
+
429
+ errors.append(message)
430
+
431
+def warning(flavor: str, distribution: str, message: str):
432
+ global warnings
433
+
434
+ message = f'{flavor}/{distribution}: {message}'
435
+ click.secho(f'Warning: {message}', fg='red')
436
+
437
+ warnings.append(message)
438
+if __name__ == "__main__":
439
+ main()
\ No newline at end of file