From 863d6ceb52d10a41b06dc3ad58fc967fc295d9b7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 21 May 2022 22:18:45 +0000 Subject: ci: fix code style In b92cb86ea14 (travis-ci: check that all build artifacts are .gitignore-d, 2017-12-31), a function was introduced with a code style that is different from the surrounding code: it added the opening curly brace on its own line, when all the existing functions in the same file cuddle that brace on the same line as the function name. Let's make the code style consistent again. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index 86e37da9bc..d718f4e386 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -69,8 +69,7 @@ skip_good_tree () { exit 0 } -check_unignored_build_artifacts () -{ +check_unignored_build_artifacts () { ! git ls-files --other --exclude-standard --error-unmatch \ -- ':/*' 2>/dev/null || { -- cgit v1.3 From b95181cf822bde2c807127abd502683822d86cf8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 21 May 2022 22:18:48 +0000 Subject: ci/run-build-and-tests: take a more high-level view In the web UI of GitHub workflows, failed runs are presented with the job step that failed auto-expanded. In the current setup, this is not helpful at all because that shows only the output of `prove`, which says which test failed, but not in what way. What would help understand the reader what went wrong is the verbose test output of the failed test. The logs of the failed runs do contain that verbose test output, but it is shown in the _next_ step (which is marked as succeeding, and is therefore _not_ auto-expanded). Anyone not intimately familiar with this would completely miss the verbose test output, being left mostly puzzled with the test failures. We are about to show the failed test cases' output in the _same_ step, so that the user has a much easier time to figure out what was going wrong. But first, we must partially revert the change that tried to improve the CI runs by combining the `Makefile` targets to build into a single `make` invocation. That might have sounded like a good idea at the time, but it does make it rather impossible for the CI script to determine whether the _build_ failed, or the _tests_. If the tests were run at all, that is. So let's go back to calling `make` for the build, and call `make test` separately so that we can easily detect that _that_ invocation failed, and react appropriately. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/run-build-and-tests.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'ci') diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 280dda7d28..2818b3046a 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -10,7 +10,7 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; *) ln -s "$cache_dir/.prove" t/.prove;; esac -export MAKE_TARGETS="all test" +run_tests=t case "$jobname" in linux-gcc) @@ -41,14 +41,15 @@ pedantic) # Don't run the tests; we only care about whether Git can be # built. export DEVOPTS=pedantic - export MAKE_TARGETS=all + run_tests= ;; esac -# Any new "test" targets should not go after this "make", but should -# adjust $MAKE_TARGETS. Otherwise compilation-only targets above will -# start running tests. -make $MAKE_TARGETS +make +if test -n "$run_tests" +then + make test +fi check_unignored_build_artifacts save_good_tree -- cgit v1.3 From 08dccc8fc1f248c8de7e87ac6e435edac4f77ebe Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 21 May 2022 22:18:49 +0000 Subject: ci: make it easier to find failed tests' logs in the GitHub workflow When investigating a test failure, the time that matters most is the time it takes from getting aware of the failure to displaying the output of the failing test case. You currently have to know a lot of implementation details when investigating test failures in the CI runs. The first step is easy: the failed job is marked quite clearly, but when opening it, the failed step is expanded, which in our case is the one running `ci/run-build-and-tests.sh`. This step, most notably, only offers a high-level view of what went wrong: it prints the output of `prove` which merely tells the reader which test script failed. The actually interesting part is in the detailed log of said failed test script. But that log is shown in the CI run's step that runs `ci/print-test-failures.sh`. And that step is _not_ expanded in the web UI by default. It is even marked as "successful", which makes it very easy to miss that there is useful information hidden in there. Let's help the reader by showing the failed tests' detailed logs in the step that is expanded automatically, i.e. directly after the test suite failed. This also helps the situation where the _build_ failed and the `print-test-failures` step was executed under the assumption that the _test suite_ failed, and consequently failed to find any failed tests. An alternative way to implement this patch would be to source `ci/print-test-failures.sh` in the `handle_test_failures` function to show these logs. However, over the course of the next few commits, we want to introduce some grouping which would be harder to achieve that way (for example, we do want a leaner, and colored, preamble for each failed test script, and it would be trickier to accommodate the lack of nested groupings in GitHub workflows' output). Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 12 ------------ ci/lib.sh | 23 +++++++++++++++++++++++ ci/run-build-and-tests.sh | 3 ++- ci/run-test-slice.sh | 3 ++- 4 files changed, 27 insertions(+), 14 deletions(-) (limited to 'ci') diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c35200defb..3fa88b78b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -119,10 +119,6 @@ jobs: - name: test shell: bash run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10 - - name: ci/print-test-failures.sh - if: failure() - shell: bash - run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v2 @@ -204,10 +200,6 @@ jobs: env: NO_SVN_TESTS: 1 run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10 - - name: ci/print-test-failures.sh - if: failure() - shell: bash - run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v2 @@ -261,8 +253,6 @@ jobs: - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - - run: ci/print-test-failures.sh - if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v2 @@ -292,8 +282,6 @@ jobs: - uses: actions/checkout@v1 - run: ci/install-docker-dependencies.sh - run: ci/run-build-and-tests.sh - - run: ci/print-test-failures.sh - if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v1 diff --git a/ci/lib.sh b/ci/lib.sh index d718f4e386..65f5188a55 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -78,6 +78,10 @@ check_unignored_build_artifacts () { } } +handle_failed_tests () { + return 1 +} + # GitHub Action doesn't set TERM, which is required by tput export TERM=${TERM:-dumb} @@ -123,6 +127,25 @@ then CI_JOB_ID="$GITHUB_RUN_ID" CC="${CC_PACKAGE:-${CC:-gcc}}" DONT_SKIP_TAGS=t + handle_failed_tests () { + mkdir -p t/failed-test-artifacts + echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV + + for test_exit in t/test-results/*.exit + do + test 0 != "$(cat "$test_exit")" || continue + + test_name="${test_exit%.exit}" + test_name="${test_name##*/}" + printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" + cat "t/test-results/$test_name.out" + + trash_dir="t/trash directory.$test_name" + cp "t/test-results/$test_name.out" t/failed-test-artifacts/ + tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + done + return 1 + } cache_dir="$HOME/none" diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 2818b3046a..1ede75e555 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -48,7 +48,8 @@ esac make if test -n "$run_tests" then - make test + make test || + handle_failed_tests fi check_unignored_build_artifacts diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index f8c2c3106a..63358c23e1 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -12,6 +12,7 @@ esac make --quiet -C t T="$(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh | - tr '\n' ' ')" + tr '\n' ' ')" || +handle_failed_tests check_unignored_build_artifacts -- cgit v1.3 From dab73aebd85acf6441769370f380f50619c3ebae Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 21 May 2022 22:18:50 +0000 Subject: ci/run-build-and-tests: add some structure to the GitHub workflow output The current output of Git's GitHub workflow can be quite confusing, especially for contributors new to the project. To make it more helpful, let's introduce some collapsible grouping. Initially, readers will see the high-level view of what actually happened (did the build fail, or the test suite?). To drill down, the respective group can be expanded. Note: sadly, workflow output currently cannot contain any nested groups (see https://github.com/actions/runner/issues/802 for details), therefore we take pains to ensure to end any previous group before starting a new one. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 56 +++++++++++++++++++++++++++++++++++++++++------ ci/run-build-and-tests.sh | 4 ++-- ci/run-test-slice.sh | 2 +- 3 files changed, 52 insertions(+), 10 deletions(-) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index 65f5188a55..f8cb79e44f 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -1,5 +1,50 @@ # Library of functions shared by all CI scripts +if test true != "$GITHUB_ACTIONS" +then + begin_group () { :; } + end_group () { :; } + + group () { + shift + "$@" + } + set -x +else + begin_group () { + need_to_end_group=t + echo "::group::$1" >&2 + set -x + } + + end_group () { + test -n "$need_to_end_group" || return 0 + set +x + need_to_end_group= + echo '::endgroup::' >&2 + } + trap end_group EXIT + + group () { + set +x + begin_group "$1" + shift + "$@" + res=$? + end_group + return $res + } + + begin_group "CI setup" +fi + +# Set 'exit on error' for all CI scripts to let the caller know that +# something went wrong. +# +# We already enabled tracing executed commands earlier. This helps by showing +# how # environment variables are set and and dependencies are installed. +set -e + skip_branch_tip_with_tag () { # Sometimes, a branch is pushed at the same time the tag that points # at the same commit as the tip of the branch is pushed, and building @@ -88,12 +133,6 @@ export TERM=${TERM:-dumb} # Clear MAKEFLAGS that may come from the outside world. export MAKEFLAGS= -# Set 'exit on error' for all CI scripts to let the caller know that -# something went wrong. -# Set tracing executed commands, primarily setting environment variables -# and installing dependencies. -set -ex - if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" then CI_TYPE=azure-pipelines @@ -138,7 +177,7 @@ then test_name="${test_exit%.exit}" test_name="${test_name##*/}" printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" - cat "t/test-results/$test_name.out" + group "Failed test: $test_name" cat "t/test-results/$test_name.out" trash_dir="t/trash directory.$test_name" cp "t/test-results/$test_name.out" t/failed-test-artifacts/ @@ -233,3 +272,6 @@ linux-leaks) esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" + +end_group +set -x diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 1ede75e555..7abfa00adc 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -45,10 +45,10 @@ pedantic) ;; esac -make +group Build make if test -n "$run_tests" then - make test || + group "Run tests" make test || handle_failed_tests fi check_unignored_build_artifacts diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index 63358c23e1..a3c67956a8 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -10,7 +10,7 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";; *) ln -s "$cache_dir/.prove" t/.prove;; esac -make --quiet -C t T="$(cd t && +group "Run tests" make --quiet -C t T="$(cd t && ./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh | tr '\n' ' ')" || handle_failed_tests -- cgit v1.3 From 0068c82a13d321b567cade0309eff8421bb2358d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 21 May 2022 22:18:54 +0000 Subject: ci: use `--github-workflow-markup` in the GitHub workflow This makes the output easier to digest. Note: since workflow output currently cannot contain any nested groups (see https://github.com/actions/runner/issues/802 for details), we need to remove the explicit grouping that would span the entirety of each failed test script. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index f8cb79e44f..de6532ee8c 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -177,7 +177,7 @@ then test_name="${test_exit%.exit}" test_name="${test_name##*/}" printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" - group "Failed test: $test_name" cat "t/test-results/$test_name.out" + cat "t/test-results/$test_name.markup" trash_dir="t/trash directory.$test_name" cp "t/test-results/$test_name.out" t/failed-test-artifacts/ @@ -189,7 +189,7 @@ then cache_dir="$HOME/none" export GIT_PROVE_OPTS="--timer --jobs 10" - export GIT_TEST_OPTS="--verbose-log -x" + export GIT_TEST_OPTS="--verbose-log -x --github-workflow-markup" MAKEFLAGS="$MAKEFLAGS --jobs=10" test windows != "$CI_OS_NAME" || GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" -- cgit v1.3 From aeea0084a0d72410422e5a4dccda5829240a5fc1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 21 May 2022 22:18:55 +0000 Subject: ci(github): mention where the full logs can be found The full logs are contained in the `failed-tests-*.zip` artifacts that are attached to the failed CI run. Since this is not immediately obvious to the well-disposed reader, let's mention it explicitly. Suggested-by: Victoria Dye Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'ci') diff --git a/ci/lib.sh b/ci/lib.sh index de6532ee8c..2f6d9d26e4 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -177,6 +177,7 @@ then test_name="${test_exit%.exit}" test_name="${test_name##*/}" printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n" + echo "The full logs are in the artifacts attached to this run." cat "t/test-results/$test_name.markup" trash_dir="t/trash directory.$test_name" -- cgit v1.3