@samitouri / QOS-React / commits / 188c1501be

Condense test262 output in ci

There's no option to output the results of the test262 harness in silent mode so every pass and failure outputs multiple lines to stdout. Since there are many thousands of tests this results in unusable log files that are over 200k lines long. This PR redirects stdout to a tmp file and then we reformat the result into a small JSON object, grouped by the failure message with count. Example: ```json [ { "pass": false, "data": { "message": "Expected no error, got Error: TODO: Support complex object assignment", "count": 66 } }, { "pass": false, "data": { "message": "Expected no error, got Error: TODO: lowerExpression(FunctionExpression)", "count": 4 } }, { "pass": false, "data": { "message": "Expected no error, got Error: TODO: lowerExpression(UnaryExpression)", "count": 6 } }, { "pass": false, "data": { "message": "Expected no error, got Error: todo: lower initializer in ForStatement", "count": 28 } }, { "pass": false, "data": { "message": "Expected no error, got Invariant Violation: Expected value for identifier `15` to be initialized.", "count": 14 } }, { "pass": false, "data": { "message": "Expected no error, got Invariant Violation: `var` declarations are not supported, use let or const", "count": 76 } }, { "pass": true, "data": { "message": null, "count": 1 } } ] ```

Lauren Tan committed Nov 16, 2022 at 18:24 UTC 188c1501be47401fbe1884a601e0e31d42a58c9a

4 files changed +57 -3

compiler/.github/workflows/test262.yml

+41 -2

@@ -5,7 +5,7 @@ on:

5 branches: [main]

7 jobs:

8 - forget:

8 + test262:

9 runs-on: ubuntu-latest

10 steps:

11 - uses: actions/checkout@v3

@@ -20,5 +20,44 @@ jobs:

20 working-directory: forget

21 - run: yarn build

22 working-directory: forget

23 - - run: yarn test262:all

23 + - id: test262

24 + name: Run test262

25 + run: echo results="$(yarn run --silent test262:ci)" >> $GITHUB_OUTPUT

26 working-directory: forget

27 + - id: format_test262_results

28 + name: Format test262 results

29 + uses: actions/github-script@v6

30 + with:

31 + script: |

32 + let pass = 0;

33 + let fail = 0;

34 + const toPercent = (a, b) => `${Math.floor((a/b) * 100)}%`;

35 + const results = JSON.parse(${{ steps.test262.outputs.results }});

36 + for (const result of results) {

37 + if (result.pass) {

38 + pass += result.count;

39 + } else {

40 + fail += result.count;

41 + }

42 + }

43 + const total = pass + fail;

44 + const comment = results

45 + .filter((result) => result.pass === false)

46 + .map((result) => `- ${result.message}: ${result.count}`)

47 + .join('\n');

48 + const totals = `Pass: ${pass} (${toPercent(pass, total)})\nFail: ${fail} (${toPercent(fail, total)})`

49 + core.setOutput("totals", totals);

50 + core.setOutput("comment", comment);

51 + - name: Create commit comment

52 + uses: peter-evans/commit-comment@v2

53 + with:

54 + body: |

55 + ## Test262 Results

56 +

57 + ### Summary

58 + ```

59 + ${{ steps.format_test262_results.outputs.totals }}

60 + ```

61 +

62 + ### Failure Details

63 + ${{ steps.format_test262_results.outputs.comment }}

compiler/forget/package.json

+2 -1

@@ -15,7 +15,8 @@

15 "test": "yarn build && jest",

16 "ts:analyze-trace": "scripts/ts-analyze-trace.sh",

17 "test262": "yarn run --silent test262-harness --preprocessor=scripts/test262-preprocessor.js",

18 - "test262:all": "yarn run --silent test262 'test262/test/**/*.js'"

18 + "test262:all": "yarn run --silent test262 'test262/test/**/*.js'",

19 + "test262:ci": "scripts/test262.sh"

20 },

21 "repository": {

22 "type": "git",

compiler/forget/scripts/test262-preprocessor.js

@@ -50,6 +50,7 @@ module.exports = (test) => {

50 throw new Error("Codegen returned an empty string");

51 }

52 } catch (error) {

53 + error.message = error.message.replace(/ $\d+:\d+$/, "");

54 test.result = {

55 stderr: `${error.name}: ${error.message}\n`,

56 stdout: "",

compiler/forget/scripts/test262.sh new

+13

@@ -0,0 +1,13 @@

1 +#!/usr/bin/env bash

2 +set -eo pipefail

3 +

4 +TMPFILE=$(mktemp /tmp/forget-test262-XXXXX)

5 +

6 +yarn run --silent test262-harness \

7 + --threads=10 \

8 + --preprocessor=scripts/test262-preprocessor.js \

9 + --reporter=json \

10 + --reporter-keys=file,result \

11 + 'test262/test/**/*.js' >> $TMPFILE

12 +

13 +jq '. | group_by(.result.pass,.result.message) | map({"pass": .[0].result.pass, "message": .[0].result.message, "count": length}) | tostring' $TMPFILE

\ No newline at end of file