From c2160f2d1943d0b72a18e21300f98ef008e48696 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 27 Jan 2019 15:26:50 -0800 Subject: ci: rename the library of common functions The name is hard-coded to reflect that we use Travis CI for continuous testing. In the next commits, we will extend this to be able use Azure DevOps, too. So let's adjust the name to make it more generic. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 ci/lib.sh (limited to 'ci/lib.sh') diff --git a/ci/lib.sh b/ci/lib.sh new file mode 100755 index 0000000000..9c6ddeb374 --- /dev/null +++ b/ci/lib.sh @@ -0,0 +1,132 @@ +# Library of functions shared by all CI scripts + +skip_branch_tip_with_tag () { + # Sometimes, a branch is pushed at the same time the tag that points + # at the same commit as the tip of the branch is pushed, and building + # both at the same time is a waste. + # + # When the build is triggered by a push to a tag, $CI_BRANCH will + # have that tagname, e.g. v2.14.0. Let's see if $CI_BRANCH is + # exactly at a tag, and if so, if it is different from $CI_BRANCH. + # That way, we can tell if we are building the tip of a branch that + # is tagged and we can skip the build because we won't be skipping a + # build of a tag. + + if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) && + test "$TAG" != "$CI_BRANCH" + then + echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)" + exit 0 + fi +} + +# Save some info about the current commit's tree, so we can skip the build +# job if we encounter the same tree again and can provide a useful info +# message. +save_good_tree () { + echo "$(git rev-parse $TRAVIS_COMMIT^{tree}) $TRAVIS_COMMIT $TRAVIS_JOB_NUMBER $TRAVIS_JOB_ID" >>"$good_trees_file" + # limit the file size + tail -1000 "$good_trees_file" >"$good_trees_file".tmp + mv "$good_trees_file".tmp "$good_trees_file" +} + +# Skip the build job if the same tree has already been built and tested +# successfully before (e.g. because the branch got rebased, changing only +# the commit messages). +skip_good_tree () { + if ! good_tree_info="$(grep "^$(git rev-parse $TRAVIS_COMMIT^{tree}) " "$good_trees_file")" + then + # Haven't seen this tree yet, or no cached good trees file yet. + # Continue the build job. + return + fi + + echo "$good_tree_info" | { + read tree prev_good_commit prev_good_job_number prev_good_job_id + + if test "$TRAVIS_JOB_ID" = "$prev_good_job_id" + then + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + This commit has already been built and tested successfully by this build job. + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + else + cat <<-EOF + $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. + The log of that build job is available at https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$prev_good_job_id + To force a re-build delete the branch's cache and then hit 'Restart job'. + EOF + fi + } + + exit 0 +} + +check_unignored_build_artifacts () +{ + ! git ls-files --other --exclude-standard --error-unmatch \ + -- ':/*' 2>/dev/null || + { + echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)" + false + } +} + +# Set 'exit on error' for all CI scripts to let the caller know that +# something went wrong. +# Set tracing executed commands, primarily setting environment variables +# and installing dependencies. +set -ex + +# When building a PR, TRAVIS_BRANCH refers to the *target* branch. Not what we +# want here. We want the source branch instead. +CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" + +cache_dir="$HOME/travis-cache" +good_trees_file="$cache_dir/good-trees" + +mkdir -p "$cache_dir" + +skip_branch_tip_with_tag +skip_good_tree + +if test -z "$jobname" +then + jobname="$TRAVIS_OS_NAME-$CC" +fi + +export DEVELOPER=1 +export DEFAULT_TEST_TARGET=prove +export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" +export GIT_TEST_OPTS="--verbose-log -x --immediate" +export GIT_TEST_CLONE_2GB=YesPlease +if [ "$jobname" = linux-gcc ]; then + export CC=gcc-8 +fi + +case "$jobname" in +linux-clang|linux-gcc) + export GIT_TEST_HTTPD=YesPlease + + # The Linux build installs the defined dependency versions below. + # The OS X build installs the latest available versions. Keep that + # in mind when you encounter a broken OS X build! + export LINUX_P4_VERSION="16.2" + export LINUX_GIT_LFS_VERSION="1.5.2" + + P4_PATH="$HOME/custom/p4" + GIT_LFS_PATH="$HOME/custom/git-lfs" + export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" + ;; +osx-clang|osx-gcc) + # t9810 occasionally fails on Travis CI OS X + # t9816 occasionally fails with "TAP out of sequence errors" on + # Travis CI OS X + export GIT_SKIP_TESTS="t9810 t9816" + ;; +GIT_TEST_GETTEXT_POISON) + export GIT_TEST_GETTEXT_POISON=YesPlease + ;; +esac -- cgit v1.3-5-g9baa From b011fabd6e07f200ed3a0a117caa409e44aba10f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 27 Jan 2019 15:26:51 -0800 Subject: ci/lib.sh: encapsulate Travis-specific things The upcoming patches will allow building git.git via Azure Pipelines (i.e. Azure DevOps' Continuous Integration), where variable names and URLs look a bit different than in Travis CI. Also, the configurations of the available agents are different. For example, Travis' and Azure Pipelines' macOS agents are set up differently, so that on Travis, we have to install the git-lfs and gettext Homebrew packages, and on Azure Pipelines we do not need to. Likewise, Azure Pipelines' Ubuntu agents already have asciidoctor installed. Finally, on Azure Pipelines the natural way is not to base64-encode tar files of the trash directories of failed tests, but to publish build artifacts instead. Therefore, that code to log those base64-encoded tar files is guarded to be Travis-specific. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 3 ++- ci/lib.sh | 45 ++++++++++++++++++++++++++++++++------------- ci/print-test-failures.sh | 8 ++++++++ ci/test-documentation.sh | 1 + 4 files changed, 43 insertions(+), 14 deletions(-) (limited to 'ci/lib.sh') diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index fe65144152..bcdcc71592 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -37,7 +37,8 @@ osx-clang|osx-gcc) brew update --quiet # Uncomment this if you want to run perf tests: # brew install gnu-time - brew install git-lfs gettext + test -z "$BREW_INSTALL_PACKAGES" || + brew install $BREW_INSTALL_PACKAGES brew link --force gettext brew install caskroom/cask/perforce ;; diff --git a/ci/lib.sh b/ci/lib.sh index 9c6ddeb374..3f286d86a6 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -24,7 +24,7 @@ skip_branch_tip_with_tag () { # job if we encounter the same tree again and can provide a useful info # message. save_good_tree () { - echo "$(git rev-parse $TRAVIS_COMMIT^{tree}) $TRAVIS_COMMIT $TRAVIS_JOB_NUMBER $TRAVIS_JOB_ID" >>"$good_trees_file" + echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file" # limit the file size tail -1000 "$good_trees_file" >"$good_trees_file".tmp mv "$good_trees_file".tmp "$good_trees_file" @@ -34,7 +34,7 @@ save_good_tree () { # successfully before (e.g. because the branch got rebased, changing only # the commit messages). skip_good_tree () { - if ! good_tree_info="$(grep "^$(git rev-parse $TRAVIS_COMMIT^{tree}) " "$good_trees_file")" + if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")" then # Haven't seen this tree yet, or no cached good trees file yet. # Continue the build job. @@ -44,18 +44,18 @@ skip_good_tree () { echo "$good_tree_info" | { read tree prev_good_commit prev_good_job_number prev_good_job_id - if test "$TRAVIS_JOB_ID" = "$prev_good_job_id" + if test "$CI_JOB_ID" = "$prev_good_job_id" then cat <<-EOF - $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) This commit has already been built and tested successfully by this build job. To force a re-build delete the branch's cache and then hit 'Restart job'. EOF else cat <<-EOF - $(tput setaf 2)Skipping build job for commit $TRAVIS_COMMIT.$(tput sgr0) + $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0) This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit. - The log of that build job is available at https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$prev_good_job_id + The log of that build job is available at $(url_for_job_id $prev_good_job_id) To force a re-build delete the branch's cache and then hit 'Restart job'. EOF fi @@ -80,11 +80,32 @@ check_unignored_build_artifacts () # and installing dependencies. set -ex -# When building a PR, TRAVIS_BRANCH refers to the *target* branch. Not what we -# want here. We want the source branch instead. -CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" +if test true = "$TRAVIS" +then + CI_TYPE=travis + # When building a PR, TRAVIS_BRANCH refers to the *target* branch. Not + # what we want here. We want the source branch instead. + CI_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}" + CI_COMMIT="$TRAVIS_COMMIT" + CI_JOB_ID="$TRAVIS_JOB_ID" + CI_JOB_NUMBER="$TRAVIS_JOB_NUMBER" + CI_OS_NAME="$TRAVIS_OS_NAME" + CI_REPO_SLUG="$TRAVIS_REPO_SLUG" + + cache_dir="$HOME/travis-cache" + + url_for_job_id () { + echo "https://travis-ci.org/$CI_REPO_SLUG/jobs/$1" + } + + BREW_INSTALL_PACKAGES="git-lfs gettext" + export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" + export GIT_TEST_OPTS="--verbose-log -x --immediate" +else + echo "Could not identify CI type" >&2 + exit 1 +fi -cache_dir="$HOME/travis-cache" good_trees_file="$cache_dir/good-trees" mkdir -p "$cache_dir" @@ -94,13 +115,11 @@ skip_good_tree if test -z "$jobname" then - jobname="$TRAVIS_OS_NAME-$CC" + jobname="$CI_OS_NAME-$CC" fi export DEVELOPER=1 export DEFAULT_TEST_TARGET=prove -export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" -export GIT_TEST_OPTS="--verbose-log -x --immediate" export GIT_TEST_CLONE_2GB=YesPlease if [ "$jobname" = linux-gcc ]; then export CC=gcc-8 diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 7aef39a2fd..cf321b474d 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -38,6 +38,14 @@ do test_name="${TEST_EXIT%.exit}" test_name="${test_name##*/}" trash_dir="trash directory.$test_name" + case "$CI_TYPE" in + travis) + ;; + *) + echo "Unhandled CI type: $CI_TYPE" >&2 + exit 1 + ;; + esac trash_tgz_b64="trash.$test_name.base64" if [ -d "$trash_dir" ] then diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh index d3cdbac73f..7d0beb2832 100755 --- a/ci/test-documentation.sh +++ b/ci/test-documentation.sh @@ -5,6 +5,7 @@ . ${0%/*}/lib.sh +test -n "$ALREADY_HAVE_ASCIIDOCTOR" || gem install asciidoctor make check-builtins -- cgit v1.3-5-g9baa From eaa62291fff35f3b33780a1572ee6e1a265adb4c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 27 Jan 2019 15:26:52 -0800 Subject: ci: inherit --jobs via MAKEFLAGS in run-build-and-tests Let's not decide in the generic ci/ part how many jobs to run in parallel; different CI configurations would favor a different number of parallel jobs, and it is easy enough to hand that information down via the `MAKEFLAGS` variable. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 1 + ci/run-build-and-tests.sh | 2 +- ci/run-linux32-build.sh | 2 +- ci/run-static-analysis.sh | 2 +- ci/test-documentation.sh | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) (limited to 'ci/lib.sh') diff --git a/ci/lib.sh b/ci/lib.sh index 3f286d86a6..32a28fd209 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -101,6 +101,7 @@ then BREW_INSTALL_PACKAGES="git-lfs gettext" export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --immediate" + export MAKEFLAGS="--jobs=2" else echo "Could not identify CI type" >&2 exit 1 diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index db342bb6a8..80d72d120f 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -7,7 +7,7 @@ ln -s "$cache_dir/.prove" t/.prove -make --jobs=2 +make make --quiet test if test "$jobname" = "linux-gcc" then diff --git a/ci/run-linux32-build.sh b/ci/run-linux32-build.sh index 2c60d2e70a..09e9276e12 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-linux32-build.sh @@ -55,6 +55,6 @@ linux32 --32bit i386 su -m -l $CI_USER -c ' set -ex cd /usr/src/git test -n "$cache_dir" && ln -s "$cache_dir/.prove" t/.prove - make --jobs=2 + make make --quiet test ' diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh index dc189c7456..a19aa7ebbc 100755 --- a/ci/run-static-analysis.sh +++ b/ci/run-static-analysis.sh @@ -5,7 +5,7 @@ . ${0%/*}/lib.sh -make --jobs=2 coccicheck +make coccicheck set +x diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh index 7d0beb2832..be3b7d376a 100755 --- a/ci/test-documentation.sh +++ b/ci/test-documentation.sh @@ -12,7 +12,7 @@ make check-builtins make check-docs # Build docs with AsciiDoc -make --jobs=2 doc > >(tee stdout.log) 2> >(tee stderr.log >&2) +make doc > >(tee stdout.log) 2> >(tee stderr.log >&2) ! test -s stderr.log test -s Documentation/git.html test -s Documentation/git.xml @@ -24,7 +24,7 @@ check_unignored_build_artifacts # Build docs with AsciiDoctor make clean -make --jobs=2 USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.log >&2) +make USE_ASCIIDOCTOR=1 doc > >(tee stdout.log) 2> >(tee stderr.log >&2) sed '/^GIT_VERSION = / d' stderr.log ! test -s stderr.log test -s Documentation/git.html -- cgit v1.3-5-g9baa From 6141a2edc9aa137d7c44a24791e50221c2ed8a5e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Jan 2019 06:19:28 -0800 Subject: ci/lib.sh: add support for Azure Pipelines This patch introduces a conditional arm that defines some environment variables and a function that displays the URL given the job id (to identify previous runs for known-good trees). Because Azure Pipeline's macOS agents already have git-lfs and gettext installed, we can leave `BREW_INSTALL_PACKAGES` empty (unlike in Travis' case). Note: this patch does not introduce an Azure Pipelines definition yet; That is left for the next patch. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 25 +++++++++++++++++++++++++ ci/print-test-failures.sh | 5 +++++ 2 files changed, 30 insertions(+) (limited to 'ci/lib.sh') diff --git a/ci/lib.sh b/ci/lib.sh index 32a28fd209..5505776876 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -102,6 +102,31 @@ then export GIT_PROVE_OPTS="--timer --jobs 3 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --immediate" export MAKEFLAGS="--jobs=2" +elif test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI" +then + CI_TYPE=azure-pipelines + # We are running in Azure Pipelines + CI_BRANCH="$BUILD_SOURCEBRANCH" + CI_COMMIT="$BUILD_SOURCEVERSION" + CI_JOB_ID="$BUILD_BUILDID" + CI_JOB_NUMBER="$BUILD_BUILDNUMBER" + CI_OS_NAME="$(echo "$AGENT_OS" | tr A-Z a-z)" + test darwin != "$CI_OS_NAME" || CI_OS_NAME=osx + CI_REPO_SLUG="$(expr "$BUILD_REPOSITORY_URI" : '.*/\([^/]*/[^/]*\)$')" + CC="${CC:-gcc}" + + # use a subdirectory of the cache dir (because the file share is shared + # among *all* phases) + cache_dir="$HOME/test-cache/$SYSTEM_PHASENAME" + + url_for_job_id () { + echo "$SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$1" + } + + BREW_INSTALL_PACKAGES= + export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" + export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" + export MAKEFLAGS="--jobs=10" else echo "Could not identify CI type" >&2 exit 1 diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index cf321b474d..e688a26f0d 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -41,6 +41,11 @@ do case "$CI_TYPE" in travis) ;; + azure-pipelines) + mkdir -p failed-test-artifacts + mv "$trash_dir" failed-test-artifacts + continue + ;; *) echo "Unhandled CI type: $CI_TYPE" >&2 exit 1 -- cgit v1.3-5-g9baa From a87e427e35fc79677dd85d591a7b37fb2539189a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Jan 2019 06:19:38 -0800 Subject: ci: speed up Windows phase As Unix shell scripting comes at a hefty price on Windows, we have to see where we can save some time to run the test suite. Let's skip the chain linting and the bin-wrappers/ redirection on Windows; this seems to shave of anywhere between 10-30% from the overall runtime. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- ci/lib.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ci/lib.sh') diff --git a/ci/lib.sh b/ci/lib.sh index 5505776876..c2bc6c68b9 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -127,6 +127,8 @@ then export GIT_PROVE_OPTS="--timer --jobs 10 --state=failed,slow,save" export GIT_TEST_OPTS="--verbose-log -x --write-junit-xml" export MAKEFLAGS="--jobs=10" + test windows_nt != "$CI_OS_NAME" || + GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" else echo "Could not identify CI type" >&2 exit 1 -- cgit v1.3-5-g9baa