| 1 | # pip install click |
| 2 | |
| 3 | import sys |
| 4 | import os |
| 5 | import re |
| 6 | import xml.etree.ElementTree |
| 7 | from xml.sax.saxutils import escape |
| 8 | |
| 9 | sys.stdout.reconfigure(encoding='utf-8', errors='replace') |
| 10 | sys.stderr.reconfigure(encoding='utf-8', errors='replace') |
| 11 | |
| 12 | def validate_line_endings(path: str, content: bytes): |
| 13 | line = 0 |
| 14 | for i in range(0, len(content)): |
| 15 | if content[i] == ord('\n'): |
| 16 | if i == 0 or content[i - 1] != ord('\r'): |
| 17 | raise RuntimeError(f'Incorrect line ending (expected CRLF) in {path}:{line}') |
| 18 | |
| 19 | line += 1 |
| 20 | |
| 21 | def get_strings_from_file(path: str, check_line_endings: bool) -> list: |
| 22 | with open(path, 'rb') as fd: |
| 23 | content = fd.read() |
| 24 | |
| 25 | if check_line_endings: |
| 26 | validate_line_endings(path, content) |
| 27 | |
| 28 | content = xml.etree.ElementTree.fromstring(content.decode()) |
| 29 | |
| 30 | result = {} |
| 31 | |
| 32 | for e in content.findall('./data'): |
| 33 | nodes = list(e.iter()) |
| 34 | |
| 35 | text = next(n.text for n in nodes if n.tag == 'value') |
| 36 | comment = next((n.text for n in nodes if n.tag == 'comment'), '') |
| 37 | name = e.get('name') |
| 38 | |
| 39 | if name in result: |
| 40 | raise RuntimeError(f'error: String "{name}" is duplicated in file "{path}"') |
| 41 | |
| 42 | result[name] = text, comment |
| 43 | |
| 44 | return result |
| 45 | |
| 46 | def cut_insert(insert: str) -> str: |
| 47 | index = 1 |
| 48 | while index < len(insert) and insert[index] != '$': |
| 49 | index += 1 |
| 50 | |
| 51 | if index + 1 >= len(insert) or insert[index] != '$': |
| 52 | raise RuntimeError(f'Invalid insert: {insert}') |
| 53 | |
| 54 | index += 1 |
| 55 | |
| 56 | if insert[index] in ['h', 'l']: |
| 57 | index += 1 |
| 58 | |
| 59 | return insert[0:index] |
| 60 | |
| 61 | def get_inserts_in_string(string: str) -> int: |
| 62 | return string.replace('{{', '').count('{') |
| 63 | |
| 64 | def get_file_string_inserts(strings: list) -> dict: |
| 65 | return {name: get_inserts_in_string(value[0]) for name, value in strings.items()} |
| 66 | |
| 67 | def validate_resource(baseline: dict, path: str): |
| 68 | strings = get_strings_from_file(path, False) |
| 69 | resource = get_file_string_inserts(strings) |
| 70 | |
| 71 | result = True |
| 72 | for string, inserts in baseline.items(): |
| 73 | if string not in resource: |
| 74 | print(f'warning: string {string} found in baseline but not in {path}') |
| 75 | continue |
| 76 | |
| 77 | if inserts != resource[string]: |
| 78 | print(f'error: Different inserts found for string {string}. Baseline: {inserts}, {path}: {resource[string]}') |
| 79 | result = False |
| 80 | |
| 81 | comment = strings[string][1] |
| 82 | locked_strings = re.findall('{Locked="([^}]*)"}', comment, re.DOTALL) |
| 83 | for e in locked_strings: |
| 84 | if e not in strings[string][0]: |
| 85 | print(f'error: locked string "{e}" not found in string {string}: {strings[string][0]}') |
| 86 | result = False |
| 87 | |
| 88 | return result |
| 89 | |
| 90 | def find_argument_end(argument: str) -> int: |
| 91 | for i in range(len(argument)): |
| 92 | if not argument[i].isalnum() and argument[i] != '-' and argument[i] != '%': |
| 93 | |
| 94 | # Include one extra character after the argument so that nothing gets added after the argument |
| 95 | # See: https://github.com/microsoft/WSL/issues/10756 |
| 96 | return min(len(argument), i + 1) |
| 97 | |
| 98 | return len(argument) |
| 99 | |
| 100 | def get_locked_strings(name: str, string: str) -> tuple[list, bool]: |
| 101 | strings = [] |
| 102 | |
| 103 | def add_arguments(prefix): |
| 104 | for i, e in enumerate(string.split(prefix)): |
| 105 | if i == 0 and not e.startswith(prefix): |
| 106 | continue |
| 107 | |
| 108 | stop_index = find_argument_end(e) |
| 109 | if stop_index > 1: |
| 110 | strings.append(prefix + e[:stop_index]) |
| 111 | |
| 112 | add_arguments('--') |
| 113 | |
| 114 | if 'wslconfig'.lower() in name.lower(): |
| 115 | add_arguments('/') # Edge case for wslconfig |
| 116 | |
| 117 | if '.wslconfig'.lower() in string.lower(): |
| 118 | strings.append('.wslconfig') |
| 119 | |
| 120 | return strings, '{}' in string |
| 121 | |
| 122 | def generate_string_comment(arguments: list, uses_insert: bool) -> str: |
| 123 | insert_rule = '{FixedPlaceholder="{}"}' if uses_insert else '' |
| 124 | return insert_rule + ''.join(f'{{Locked="{e}"}}' for e in arguments) + 'Command line arguments, file names and string inserts should not be translated' |
| 125 | |
| 126 | def validate_comments(strings: dict): |
| 127 | result = True |
| 128 | |
| 129 | comments_changes = {} |
| 130 | for name, (string, comment) in strings.items(): |
| 131 | arguments, uses_insert = get_locked_strings(name, string) |
| 132 | |
| 133 | if len(arguments) == 0 and not uses_insert: |
| 134 | continue # No command line arguments or inserts in this string |
| 135 | |
| 136 | # For the sake of simplicity this logic makes the assumption that comments |
| 137 | # are always in the same order as of the original string |
| 138 | expected_comment = generate_string_comment(arguments, uses_insert) |
| 139 | if not expected_comment in comment: |
| 140 | comments_changes[name] = (comment, expected_comment) |
| 141 | print(f'Incorrect comment for string {name}. Expected comment: <comment>{expected_comment}</comment>') |
| 142 | result = False |
| 143 | |
| 144 | return result, comments_changes |
| 145 | |
| 146 | def fix_comments(comments: dict, path: str, strings: dict): |
| 147 | with open(path, 'rb') as fd: |
| 148 | content = fd.read() |
| 149 | |
| 150 | missed = 0 |
| 151 | for name, (comment, fixed_comment) in comments.items(): |
| 152 | comment = comment.replace('\n', '\r\n') |
| 153 | matches = content.count(comment.encode()) |
| 154 | if comment and matches == 1: |
| 155 | content = content.replace(comment.encode(), fixed_comment.encode()) |
| 156 | continue |
| 157 | elif not comment or matches == 0: |
| 158 | # Try to add the comment if it doesn't exist at all |
| 159 | reconstructed_xml = f''' <data name="{name}" xml:space="preserve"> |
| 160 | <value>{escape(strings[name][0])}</value> |
| 161 | ''' |
| 162 | suffix = ' </data>' |
| 163 | pattern = (reconstructed_xml + suffix).replace('\n', '\r\n').encode() |
| 164 | if content.count(pattern) == 1: |
| 165 | content = content.replace( |
| 166 | pattern, |
| 167 | f'{reconstructed_xml} <comment>{fixed_comment}</comment>\n{suffix}'.replace('\n', '\r\n').encode()) |
| 168 | |
| 169 | continue |
| 170 | |
| 171 | click.secho(f"Couldn't find unique match for comment (name={name}): {comment}. It needs to be manually replaced with: {fixed_comment}") |
| 172 | missed += 1 |
| 173 | |
| 174 | with open(path, 'wb') as fd: |
| 175 | fd.write(content) |
| 176 | |
| 177 | click.secho(f'Updated file: {path}. {missed} comments need manual changes', fg='green' if missed == 0 else 'yellow', bold=True) |
| 178 | |
| 179 | |
| 180 | ADML_NS = '{http://schemas.microsoft.com/GroupPolicy/2006/07/PolicyDefinitions}' |
| 181 | RESOURCE_FOLDER = 'localization/strings' |
| 182 | BASELINE_LANGUAGE = 'en-US' |
| 183 | ADML_FOLDER = 'intune' |
| 184 | ADML_FILENAME = 'WSL.adml' |
| 185 | |
| 186 | def get_adml_entries(path: str) -> tuple[dict, set]: |
| 187 | """Parse an .adml file. |
| 188 | |
| 189 | Returns ({string_id: (value, [locked_tokens])}, {presentation_id, ...}). |
| 190 | Locked tokens are extracted from inline comments inside <string> elements: |
| 191 | `<string id="X"><!-- _locComment='{Locked="..."}' -->...text...</string>` |
| 192 | The whole-string `{Locked}` form is also recognized but contributes no |
| 193 | specific tokens to verify. This is the only form honored by the Touchdown |
| 194 | POMXML parser; standalone comments preceding a <string> are ignored by |
| 195 | Touchdown and so are not recognized here either. |
| 196 | """ |
| 197 | # Parse with a TreeBuilder that preserves comments so we can associate |
| 198 | # {Locked="..."} tokens with the <string> element they belong to. |
| 199 | parser = xml.etree.ElementTree.XMLParser( |
| 200 | target=xml.etree.ElementTree.TreeBuilder(insert_comments=True)) |
| 201 | root = xml.etree.ElementTree.parse(path, parser=parser).getroot() |
| 202 | |
| 203 | string_table = root.find(f'.//{ADML_NS}stringTable') |
| 204 | if string_table is None: |
| 205 | raise RuntimeError(f'error: {path} is missing the required <stringTable> element') |
| 206 | |
| 207 | strings = {} |
| 208 | for child in string_table: |
| 209 | if child.tag != f'{ADML_NS}string': |
| 210 | continue |
| 211 | sid = child.get('id') |
| 212 | if sid is None: |
| 213 | continue |
| 214 | # The string value is the text before the first child, plus the tail of |
| 215 | # any inline comment children (which is where the actual visible text |
| 216 | # ends up when the comment precedes it inside the <string>). |
| 217 | value = (child.text or '') + ''.join((c.tail or '') for c in child) |
| 218 | tokens = [] |
| 219 | for c in child: |
| 220 | if c.tag is xml.etree.ElementTree.Comment: |
| 221 | tokens.extend(re.findall(r'\{Locked="([^"]*)"\}', c.text or '')) |
| 222 | strings[sid] = (value, tokens) |
| 223 | |
| 224 | presentation_table = root.find(f'.//{ADML_NS}presentationTable') |
| 225 | if presentation_table is None: |
| 226 | raise RuntimeError(f'error: {path} is missing the required <presentationTable> element') |
| 227 | |
| 228 | presentations = {p.get('id') for p in presentation_table.findall(f'{ADML_NS}presentation') if p.get('id')} |
| 229 | |
| 230 | return strings, presentations |
| 231 | |
| 232 | def validate_adml(adml_folder: str, baseline_language: str) -> bool: |
| 233 | baseline_path = f'{adml_folder}/{baseline_language}/{ADML_FILENAME}' |
| 234 | if not os.path.isfile(baseline_path): |
| 235 | print(f'info: ADML baseline not found at {baseline_path}, skipping ADML validation') |
| 236 | return True |
| 237 | |
| 238 | print(f'Validating ADML baseline {baseline_path}') |
| 239 | baseline, baseline_presentations = get_adml_entries(baseline_path) |
| 240 | baseline_ids = set(baseline.keys()) |
| 241 | |
| 242 | result = True |
| 243 | for sid, (value, tokens) in baseline.items(): |
| 244 | for tok in tokens: |
| 245 | if tok not in value: |
| 246 | print(f'error: locked token "{tok}" not found in baseline ADML string {sid}: {value}') |
| 247 | result = False |
| 248 | |
| 249 | if not os.path.isdir(adml_folder): |
| 250 | return result |
| 251 | |
| 252 | for entry in sorted(os.listdir(adml_folder)): |
| 253 | locale_path = f'{adml_folder}/{entry}/{ADML_FILENAME}' |
| 254 | if entry == baseline_language or not os.path.isfile(locale_path): |
| 255 | continue |
| 256 | |
| 257 | print(f'Validating ADML {locale_path}') |
| 258 | translated, translated_presentations = get_adml_entries(locale_path) |
| 259 | |
| 260 | missing = baseline_ids - set(translated.keys()) |
| 261 | extra = set(translated.keys()) - baseline_ids |
| 262 | if missing: |
| 263 | print(f'error: ADML {locale_path} is missing string ids: {sorted(missing)}') |
| 264 | result = False |
| 265 | if extra: |
| 266 | print(f'error: ADML {locale_path} has unexpected string ids: {sorted(extra)}') |
| 267 | result = False |
| 268 | |
| 269 | missing_p = baseline_presentations - translated_presentations |
| 270 | extra_p = translated_presentations - baseline_presentations |
| 271 | if missing_p: |
| 272 | print(f'error: ADML {locale_path} is missing presentation ids: {sorted(missing_p)}') |
| 273 | result = False |
| 274 | if extra_p: |
| 275 | print(f'error: ADML {locale_path} has unexpected presentation ids: {sorted(extra_p)}') |
| 276 | result = False |
| 277 | |
| 278 | # Note: we intentionally do not enforce that baseline {Locked="..."} |
| 279 | # tokens appear in translated ADML strings. Translated locale files |
| 280 | # generated before the en-US source migrated to inline _locComment |
| 281 | # directives still contain translated tokens; failing CI here would |
| 282 | # block every nightly localization PR until those caches refresh. The |
| 283 | # baseline check above still catches authoring mistakes in en-US. |
| 284 | for sid in baseline_ids & set(translated.keys()): |
| 285 | _, tokens = baseline[sid] |
| 286 | tvalue, _ = translated[sid] |
| 287 | for tok in tokens: |
| 288 | if tok not in tvalue: |
| 289 | print(f'warning: locked token "{tok}" not preserved in {locale_path} string {sid}: {tvalue}') |
| 290 | |
| 291 | return result |
| 292 | |
| 293 | def run(resource_folder: str, baseline_language: str, fix: bool, adml_folder: str): |
| 294 | baseline_file = f'{resource_folder}/{baseline_language}/Resources.resw' |
| 295 | |
| 296 | strings = get_strings_from_file(baseline_file, True) |
| 297 | baseline = get_file_string_inserts(strings) |
| 298 | |
| 299 | result, comments = validate_comments(strings) |
| 300 | for language in os.listdir(resource_folder): |
| 301 | path = f'{resource_folder}/{language}/Resources.resw' |
| 302 | print(f'Validating inserts in {path}') |
| 303 | result &= validate_resource(baseline, path) |
| 304 | |
| 305 | result &= validate_adml(adml_folder, baseline_language) |
| 306 | |
| 307 | if fix and comments: |
| 308 | fix_comments(comments, baseline_file, strings) |
| 309 | |
| 310 | sys.exit(0 if result else 1) |
| 311 | |
| 312 | |
| 313 | if __name__ == '__main__': |
| 314 | if len(sys.argv) == 1: # Avoid pulling in click for the default (CI) invocation |
| 315 | run(RESOURCE_FOLDER, BASELINE_LANGUAGE, False, ADML_FOLDER) |
| 316 | else: |
| 317 | import click |
| 318 | |
| 319 | @click.command() |
| 320 | @click.option('--resource-folder', default=RESOURCE_FOLDER, show_default=True) |
| 321 | @click.option('--baseline-language', default=BASELINE_LANGUAGE, show_default=True) |
| 322 | @click.option('--adml-folder', default=ADML_FOLDER, show_default=True) |
| 323 | @click.option('--fix', is_flag=True) |
| 324 | def main(resource_folder: str, baseline_language: str, adml_folder: str, fix: bool): |
| 325 | run(resource_folder, baseline_language, fix, adml_folder) |
| 326 | |
| 327 | main() |