tools/travis-ci/run.sh

   1 #!/bin/bash
   2
   3 # This script executes the script step when running under travis-ci
   4
   5 #if cygwin, check path
   6 case ${MACHTYPE} in
   7         *cygwin*) OPJ_CI_IS_CYGWIN=1;;
   8         *) ;;
   9 esac
  10
  11 # Hack for appveyor to get GNU find in path before windows one.
  12 export PATH=$(dirname ${BASH}):$PATH
  13
  14 # Set-up some bash options
  15 set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
  16 set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
  17 set -o pipefail  ## Fail on error in pipe
  18
  19 function opjpath ()
  20 {
  21         if [ "${OPJ_CI_IS_CYGWIN:-}" == "1" ]; then
  22                 cygpath $1 "$2"
  23         else
  24                 echo "$2"
  25         fi
  26 }
  27
  28 # ABI check is done by abi-check.sh
  29 if [ "${OPJ_CI_ABI_CHECK:-}" == "1" ]; then
  30         exit 0
  31 fi
  32
  33 if [ "${OPJ_CI_CC:-}" != "" ]; then
  34     export CC=${OPJ_CI_CC}
  35     echo "Using ${CC}"
  36 fi
  37
  38 if [ "${OPJ_CI_CXX:-}" != "" ]; then
  39     export CXX=${OPJ_CI_CXX}
  40     echo "Using ${CXX}"
  41 fi
  42
  43 # Set-up some variables
  44 if [ "${OPJ_CI_BUILD_CONFIGURATION:-}" == "" ]; then
  45         export OPJ_CI_BUILD_CONFIGURATION=Release #default
  46 fi
  47 OPJ_SOURCE_DIR=$(cd $(dirname $0)/../.. && pwd)
  48
  49 if [ "${OPJ_DO_SUBMIT:-}" == "" ]; then
  50         OPJ_DO_SUBMIT=0 # Do not flood cdash by default
  51 fi
  52
  53 if [ "${GITHUB_REPOSITORY:-}" != "" ]; then
  54         OPJ_OWNER=$(echo "${GITHUB_REPOSITORY}" | sed 's/\(^.*\)\/.*/\1/')
  55         OPJ_SITE="${OPJ_OWNER}.gha"
  56 elif [ "${TRAVIS_REPO_SLUG:-}" != "" ]; then
  57         OPJ_OWNER=$(echo "${TRAVIS_REPO_SLUG}" | sed 's/\(^.*\)\/.*/\1/')
  58         OPJ_SITE="${OPJ_OWNER}.travis-ci.org"
  59 elif [ "${APPVEYOR_REPO_NAME:-}" != "" ]; then
  60         OPJ_OWNER=$(echo "${APPVEYOR_REPO_NAME}" | sed 's/\(^.*\)\/.*/\1/')
  61         OPJ_SITE="${OPJ_OWNER}.appveyor.com"
  62 else
  63         OPJ_SITE="$(hostname)"
  64 fi
  65
  66 if [ "${OPJ_OWNER:-}" == "uclouvain" ]; then
  67     OPJ_DO_SUBMIT=1
  68 fi
  69
  70 if [ "${RUNNER_OS:-}" != "" ]; then
  71     if [ "${RUNNER_OS:-}" == "Linux" ]; then
  72         OPJ_SHORT_OS_NAME=linux
  73     elif [ "${RUNNER_OS:-}" == "Windows" ]; then
  74         OPJ_SHORT_OS_NAME=windows
  75     elif [ "${RUNNER_OS:-}" == "macOS" ]; then
  76         OPJ_SHORT_OS_NAME=osx
  77     else
  78         echo "Unhandled RUNNER_OS = ${RUNNER_OS:-}"; exit 1
  79     fi
  80 elif [ "${TRAVIS_OS_NAME:-}" != "" ]; then
  81   OPJ_SHORT_OS_NAME="${${TRAVIS_OS_NAME:-}}"
  82 else
  83   # Let's guess OS for testing purposes
  84         echo "Guessing OS"
  85         if uname -s | grep -i Darwin &> /dev/null; then
  86                 OPJ_SHORT_OS_NAME=osx
  87         elif uname -s | grep -i Linux &> /dev/null; then
  88                 OPJ_SHORT_OS_NAME=linux
  89         elif uname -s | grep -i CYGWIN &> /dev/null; then
  90                 OPJ_SHORT_OS_NAME=windows
  91         elif uname -s | grep -i MINGW &> /dev/null; then
  92                 OPJ_SHORT_OS_NAME=windows
  93         elif [ "${APPVEYOR:-}" == "True" ]; then
  94                 OPJ_SHORT_OS_NAME=windows
  95         else
  96                 echo "Failed to guess OS"; exit 1
  97         fi
  98         echo "${OPJ_SHORT_OS_NAME}"
  99 fi
 100
 101 if [ "${OPJ_SHORT_OS_NAME}" == "osx" ]; then
 102         OPJ_OS_NAME=$(sw_vers -productName | tr -d ' ')$(sw_vers -productVersion | sed 's/\([^0-9]*\.[0-9]*\).*/\1/')
 103         OPJ_CC_VERSION=$(xcodebuild -version | grep -i xcode)
 104         OPJ_CC_VERSION=xcode${OPJ_CC_VERSION:6}
 105 elif [ "${OPJ_SHORT_OS_NAME}" == "linux" ]; then
 106         OPJ_OS_NAME=linux
 107         if which lsb_release > /dev/null; then
 108                 OPJ_OS_NAME=$(lsb_release -si)$(lsb_release -sr | sed 's/\([^0-9]*\.[0-9]*\).*/\1/')
 109         fi
 110         if [ "${CC:-}" == "" ]; then
 111                 # default to gcc
 112                 export CC=gcc
 113                 echo "Defaulting to CC=gcc"
 114         fi
 115         if [ -z "${CC##*gcc*}" ]; then
 116                 OPJ_CC_VERSION=$(${CC} --version | head -1 | sed 's/.*\ \([0-9.]*[0-9]\)/\1/')
 117                 if [ -z "${CC##*mingw*}" ]; then
 118                         OPJ_CC_VERSION=mingw${OPJ_CC_VERSION}
 119                         # disable testing for now
 120                         export OPJ_CI_SKIP_TESTS=1
 121                 else
 122                         OPJ_CC_VERSION=gcc${OPJ_CC_VERSION}
 123                 fi
 124         elif [ -z "${CC##*clang*}" ]; then
 125                 OPJ_CC_VERSION=clang$(${CC} --version | grep version | sed 's/.*version \([^0-9.]*[0-9.]*\).*/\1/')
 126         else
 127                 echo "Compiler not supported: ${CC}"; exit 1
 128         fi
 129         if [ "${OPJ_CI_INSTRUCTION_SETS-:}" == "-mavx2" ]; then
 130                 AVX2_AVAIL=1
 131                 cat /proc/cpuinfo | grep avx2 >/dev/null || AVX2_AVAIL=0
 132                 if [[ "${AVX2_AVAIL}" == "1" ]]; then
 133                         echo "AVX2 available on CPU"
 134                 else
 135                         echo "AVX2 not available on CPU. Disabling tests"
 136                         cat /proc/cpuinfo  | grep flags | head -n 1
 137                         export OPJ_CI_SKIP_TESTS=1
 138                 fi
 139         fi
 140 elif [ "${OPJ_SHORT_OS_NAME}" == "windows" ]; then
 141         OPJ_OS_NAME=windows
 142         if which cl > /dev/null; then
 143                 OPJ_CL_VERSION=$(cl 2>&1 | grep Version | sed 's/.*Version \([0-9]*\).*/\1/')
 144                 if [ ${OPJ_CL_VERSION} -eq 19 ]; then
 145                         OPJ_CC_VERSION=vs2015
 146                 elif [ ${OPJ_CL_VERSION} -eq 18 ]; then
 147                         OPJ_CC_VERSION=vs2013
 148                 elif [ ${OPJ_CL_VERSION} -eq 17 ]; then
 149                         OPJ_CC_VERSION=vs2012
 150                 elif [ ${OPJ_CL_VERSION} -eq 16 ]; then
 151                         OPJ_CC_VERSION=vs2010
 152                 elif [ ${OPJ_CL_VERSION} -eq 15 ]; then
 153                         OPJ_CC_VERSION=vs2008
 154                 elif [ ${OPJ_CL_VERSION} -eq 14 ]; then
 155                         OPJ_CC_VERSION=vs2005
 156                 else
 157                         OPJ_CC_VERSION=vs????
 158                 fi
 159         fi
 160         if [ "${OPJ_CI_INSTRUCTION_SETS-:}" == "/arch:AVX2" ]; then
 161                 cl $PWD/tools/travis-ci/detect-avx2.c
 162                 if ./detect-avx2.exe; then
 163                         echo "AVX2 available on CPU"
 164                 else
 165                         echo "AVX2 not available on CPU. Disabling tests"
 166                         export OPJ_CI_SKIP_TESTS=1
 167                 fi
 168         fi
 169 else
 170         echo "OS not supported: ${OPJ_SHORT_OS_NAME}"; exit 1
 171 fi
 172
 173 if [ "${OPJ_CI_ARCH:-}" == "" ]; then
 174         echo "Guessing build architecture"
 175         MACHINE_ARCH=$(uname -m)
 176         if [ "${MACHINE_ARCH}" == "x86_64" ]; then
 177                 export OPJ_CI_ARCH=x86_64
 178         fi
 179         echo "${OPJ_CI_ARCH}"
 180 fi
 181
 182 if [ "${GITHUB_HEAD_REF:-}" != "" ]; then
 183     OPJ_BRANCH=${GITHUB_HEAD_REF}
 184 elif [ "${TRAVIS_BRANCH:-}" != "" ]; then
 185     OPJ_BRANCH=${TRAVIS_BRANCH}
 186 elif [ "${APPVEYOR_REPO_BRANCH:-}" != "" ]; then
 187     OPJ_BRANCH=${APPVEYOR_REPO_BRANCH}
 188 else
 189     echo "Guessing branch"
 190     OPJ_BRANCH=$(git -C ${OPJ_SOURCE_DIR} branch | grep '*' | tr -d '*[[:blank:]]')
 191 fi
 192
 193 OPJ_BUILDNAME=${OPJ_OS_NAME}-${OPJ_CC_VERSION}-${OPJ_CI_ARCH}-${OPJ_BRANCH}
 194 OPJ_BUILDNAME_TEST=${OPJ_OS_NAME}-${OPJ_CC_VERSION}-${OPJ_CI_ARCH}
 195 if [ "${OPJ_CI_INSTRUCTION_SETS-:}" == "/arch:AVX2" ]; then
 196         OPJ_BUILDNAME=${OPJ_BUILDNAME}-avx2
 197         OPJ_BUILDNAME_TEST=${OPJ_BUILDNAME_TEST}-avx2
 198 fi
 199 if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then
 200     PULL_REQUEST_NUMBER=$(echo $GITHUB_REF | awk 'BEGIN { FS = "/" } ; { print $3 }')
 201         OPJ_BUILDNAME=${OPJ_BUILDNAME}-${PULL_REQUEST_NUMBER}
 202 elif [ "${TRAVIS_PULL_REQUEST:-}" != "false" ] && [ "${TRAVIS_PULL_REQUEST:-}" != "" ]; then
 203         OPJ_BUILDNAME=${OPJ_BUILDNAME}-pr${TRAVIS_PULL_REQUEST}
 204 elif [ "${APPVEYOR_PULL_REQUEST_NUMBER:-}" != "" ]; then
 205         OPJ_BUILDNAME=${OPJ_BUILDNAME}-pr${APPVEYOR_PULL_REQUEST_NUMBER}
 206 fi
 207 OPJ_BUILDNAME=${OPJ_BUILDNAME}-${OPJ_CI_BUILD_CONFIGURATION}-3rdP
 208 OPJ_BUILDNAME_TEST=${OPJ_BUILDNAME_TEST}-${OPJ_CI_BUILD_CONFIGURATION}-3rdP
 209 if [ "${OPJ_CI_ASAN:-}" == "1" ]; then
 210         OPJ_BUILDNAME=${OPJ_BUILDNAME}-ASan
 211         OPJ_BUILDNAME_TEST=${OPJ_BUILDNAME_TEST}-ASan
 212 fi
 213
 214 if [ "${OPJ_NONCOMMERCIAL:-}" == "1" ] && [ "${OPJ_CI_SKIP_TESTS:-}" != "1" ] && [ -d kdu ]; then
 215         echo "
 216 Testing will use Kakadu trial binaries. Here's the copyright notice from kakadu:
 217 Copyright is owned by NewSouth Innovations Pty Limited, commercial arm of the UNSW Australia in Sydney.
 218 You are free to trial these executables and even to re-distribute them,
 219 so long as such use or re-distribution is accompanied with this copyright notice and is not for commercial gain.
 220 Note: Binaries can only be used for non-commercial purposes.
 221 "
 222 fi
 223
 224 set -x
 225 # This will print configuration
 226 # travis-ci doesn't dump cmake version in system info, let's print it
 227 cmake --version
 228
 229 export OPJ_SHORT_OS_NAME=${OPJ_SHORT_OS_NAME}
 230 export OPJ_SITE=${OPJ_SITE}
 231 export OPJ_BUILDNAME=${OPJ_BUILDNAME}
 232 export OPJ_SOURCE_DIR=$(opjpath -m ${OPJ_SOURCE_DIR})
 233 export OPJ_BINARY_DIR=$(opjpath -m ${PWD}/build)
 234 export OPJ_BUILD_CONFIGURATION=${OPJ_CI_BUILD_CONFIGURATION}
 235 export OPJ_DO_SUBMIT=${OPJ_DO_SUBMIT}
 236
 237 if [ "${OPJ_SKIP_REBUILD:-}" != "1" ]; then
 238     ctest -S ${OPJ_SOURCE_DIR}/tools/ctest_scripts/travis-ci.cmake -V || true
 239 fi
 240 # ctest will exit with various error codes depending on version.
 241 # ignore ctest exit code & parse this ourselves
 242 set +x
 243
 244
 245
 246 if [ "${OPJ_CI_CHECK_STYLE:-}" == "1" ]; then
 247     export OPJSTYLE=${PWD}/scripts/opjstyle
 248     export PATH=${HOME}/.local/bin:${PATH}
 249     scripts/verify-indentation.sh
 250 fi
 251
 252
 253 # Deployment if needed
 254 #---------------------
 255 if [ "${TRAVIS_TAG:-}" != "" ]; then
 256     OPJ_TAG_NAME=${TRAVIS_TAG}
 257 elif [ "${APPVEYOR_REPO_TAG:-}" == "true" ]; then
 258     OPJ_TAG_NAME=${APPVEYOR_REPO_TAG_NAME}
 259 elif test $(git describe --exact-match --tags 2>/dev/null); then
 260     OPJ_TAG_NAME="$(git describe --exact-match --tags)"
 261 else
 262     OPJ_TAG_NAME=""
 263 fi
 264 if [ "${OPJ_CI_INCLUDE_IF_DEPLOY:-}" == "1" ] && [ "${OPJ_TAG_NAME:-}" != "" ]; then
 265 #if [ "${OPJ_CI_INCLUDE_IF_DEPLOY:-}" == "1" ]; then
 266         OPJ_CI_DEPLOY=1         # unused for now
 267         OPJ_CUR_DIR=${PWD}
 268         if [ "${OPJ_SHORT_OS_NAME:-}" == "linux" ]; then
 269                 OPJ_PACK_GENERATOR="TGZ" # ZIP generator currently segfaults on linux
 270         else
 271                 OPJ_PACK_GENERATOR="ZIP"
 272         fi
 273         OPJ_PACK_NAME="openjpeg-${OPJ_TAG_NAME}-${OPJ_SHORT_OS_NAME}-${OPJ_CI_ARCH}"
 274         cd ${OPJ_BINARY_DIR}
 275         cmake -D CPACK_GENERATOR:STRING=${OPJ_PACK_GENERATOR} -D CPACK_PACKAGE_FILE_NAME:STRING=${OPJ_PACK_NAME} ${OPJ_SOURCE_DIR}
 276         cd ${OPJ_CUR_DIR}
 277         cmake --build ${OPJ_BINARY_DIR} --target package
 278         echo "ready to deploy $(ls ${OPJ_BINARY_DIR}/${OPJ_PACK_NAME}*) to GitHub releases"
 279         if [ "${APPVEYOR_REPO_TAG:-}" == "true" ]; then
 280                 appveyor PushArtifact "${OPJ_BINARY_DIR}/${OPJ_PACK_NAME}.zip"
 281         fi
 282 else
 283         OPJ_CI_DEPLOY=0
 284 fi
 285
 286 # let's parse configure/build/tests for failure
 287
 288 echo "
 289 Parsing logs for failures
 290 "
 291 OPJ_CI_RESULT=0
 292
 293 # 1st configure step
 294 OPJ_CONFIGURE_XML=$(find build -path 'build/Testing/*' -name 'Configure.xml')
 295 if [ ! -f "${OPJ_CONFIGURE_XML}" ]; then
 296         echo "No configure log found"
 297         OPJ_CI_RESULT=1
 298 else
 299         if ! grep '<ConfigureStatus>0</ConfigureStatus>' ${OPJ_CONFIGURE_XML} &> /dev/null; then
 300                 echo "Errors were found in configure log"
 301                 OPJ_CI_RESULT=1
 302         fi
 303 fi
 304
 305 # 2nd build step
 306 # We must have one Build.xml file
 307 OPJ_BUILD_XML=$(find build -path 'build/Testing/*' -name 'Build.xml')
 308 if [ ! -f "${OPJ_BUILD_XML}" ]; then
 309         echo "No build log found"
 310         OPJ_CI_RESULT=1
 311 else
 312         if grep '<Error>' ${OPJ_BUILD_XML} &> /dev/null; then
 313                 echo "Errors were found in build log"
 314                 OPJ_CI_RESULT=1
 315         fi
 316 fi
 317
 318 if [ ${OPJ_CI_RESULT} -ne 0 ]; then
 319         # Don't trash output with failing tests when there are configure/build errors
 320         exit ${OPJ_CI_RESULT}
 321 fi
 322
 323 if [ "${OPJ_CI_SKIP_TESTS:-}" != "1" ]; then
 324         OPJ_TEST_XML=$(find build -path 'build/Testing/*' -name 'Test.xml')
 325         if [ ! -f "${OPJ_TEST_XML}" ]; then
 326                 echo "No test log found"
 327                 OPJ_CI_RESULT=1
 328         else
 329                 echo "Parsing tests for new/unknown failures"
 330                 # 3rd test step
 331                 OPJ_FAILEDTEST_LOG=$(find build -path 'build/Testing/Temporary/*' -name 'LastTestsFailed_*.log')
 332                 if [ -f "${OPJ_FAILEDTEST_LOG}" ]; then
 333                         awk -F: '{ print $2 }' ${OPJ_FAILEDTEST_LOG} > failures.txt
 334                         while read FAILEDTEST; do
 335                                 # Start with common errors
 336                                 if grep -x "${FAILEDTEST}" $(opjpath -u ${OPJ_SOURCE_DIR})/tools/travis-ci/knownfailures-all.txt > /dev/null; then
 337                                         continue
 338                                 fi
 339                                 if [ -f $(opjpath -u ${OPJ_SOURCE_DIR})/tools/travis-ci/knownfailures-${OPJ_BUILDNAME_TEST}.txt ]; then
 340                                         if grep -x "${FAILEDTEST}" $(opjpath -u ${OPJ_SOURCE_DIR})/tools/travis-ci/knownfailures-${OPJ_BUILDNAME_TEST}.txt > /dev/null; then
 341                                                 continue
 342                                         fi
 343                                 fi
 344                                 echo "${FAILEDTEST}"
 345                                 OPJ_CI_RESULT=1
 346                         done < failures.txt
 347                 fi
 348         fi
 349
 350         if [ ${OPJ_CI_RESULT} -eq 0 ]; then
 351                 echo "No new/unknown test failure found
 352                 "
 353         else
 354                 echo "
 355 New/unknown test failure found!!! You may need to update/create tools/travis-ci/knownfailures-${OPJ_BUILDNAME_TEST}.txt
 356         "
 357         fi
 358
 359         # 4th memcheck step
 360         OPJ_MEMCHECK_XML=$(find build -path 'build/Testing/*' -name 'DynamicAnalysis.xml')
 361         if [ -f "${OPJ_MEMCHECK_XML}" ]; then
 362                 if grep '<Defect Type' ${OPJ_MEMCHECK_XML} 2> /dev/null; then
 363                         echo "Errors were found in dynamic analysis log"
 364                         OPJ_CI_RESULT=1
 365                 fi
 366         fi
 367 fi
 368
 369 if [ "${OPJ_CI_BUILD_FUZZERS:-}" == "1" ]; then
 370     cd tests/fuzzers
 371     make
 372     cd ../..
 373 fi
 374
 375 if [ "${OPJ_CI_PERF_TESTS:-}" == "1" ]; then
 376     cd tests/performance
 377     echo "Running performance tests on current version (dry-run)"
 378     PATH=../../build/bin:$PATH python ./perf_test.py
 379     echo "Running performance tests on current version"
 380     PATH=../../build/bin:$PATH python ./perf_test.py -o /tmp/new.csv
 381     if [ "${OPJ_NONCOMMERCIAL:-}" == "1" ] && [ -d ../../kdu ]; then
 382         echo "Running performances tests with Kakadu"
 383         LD_LIBRARY_PATH=../../kdu PATH=../../kdu::$PATH python ./perf_test.py -kakadu -o /tmp/kakadu.csv
 384         echo "Comparing current version with Kakadu"
 385         python compare_perfs.py /tmp/kakadu.csv /tmp/new.csv || true
 386     fi
 387     cd ../..
 388
 389     REF_VERSION=master
 390     if [ "${TRAVIS_PULL_REQUEST:-false}" == "false" -a "${GITHUB_EVENT_NAME:-}" != "pull_request" ]; then
 391         REF_VERSION=v2.1.2
 392     fi
 393     if [ ! -d ref_opj ]; then
 394         git clone https://github.com/uclouvain/openjpeg ref_opj
 395     fi
 396     echo "Building reference version (${REF_VERSION})"
 397     cd ref_opj
 398     git checkout ${REF_VERSION}
 399     mkdir -p build
 400     cd build
 401     cmake .. -DCMAKE_BUILD_TYPE=${OPJ_BUILD_CONFIGURATION}
 402     make -j3
 403     cd ../..
 404     cd tests/performance
 405     echo "Running performance tests on ${REF_VERSION} version (dry-run)"
 406     PATH=../../ref_opj/build/bin:$PATH python ./perf_test.py
 407     echo "Running performance tests on ${REF_VERSION} version"
 408     PATH=../../ref_opj/build/bin:$PATH python ./perf_test.py -o /tmp/ref.csv
 409     echo "Comparing current version with ${REF_VERSION} version"
 410     # we should normally set OPJ_CI_RESULT=1 in case of failure, but
 411     # this is too unreliable
 412     python compare_perfs.py /tmp/ref.csv /tmp/new.csv || true
 413     cd ../..
 414 fi
 415
 416 if [ "${OPJ_CI_PROFILE:-}" == "1" ]; then
 417     rm -rf build_gprof
 418     mkdir build_gprof
 419     cd build_gprof
 420     # We need static linking for gprof
 421     cmake "-DCMAKE_C_FLAGS=-pg -O3" -DCMAKE_EXE_LINKER_FLAGS=-pg -DCMAKE_SHARED_LINKER_FLAGS=-pg -DBUILD_SHARED_LIBS=OFF ..
 422     make -j3
 423     cd ..
 424     build_gprof/bin/opj_decompress -i data/input/nonregression/kodak_2layers_lrcp.j2c -o out.tif > /dev/null
 425     echo "Most CPU consuming functions:"
 426     gprof build_gprof/bin/opj_decompress gmon.out | head || true
 427
 428     rm -f massif.out.*
 429     valgrind --tool=massif build/bin/opj_decompress -i data/input/nonregression/kodak_2layers_lrcp.j2c -o out.tif >/dev/null 2>/dev/null
 430     echo ""
 431     echo "Memory consumption profile:"
 432     python tests/profiling/filter_massif_output.py massif.out.*
 433 fi
 434
 435 exit ${OPJ_CI_RESULT}