tools/travis-ci/run.sh

   1 #!/bin/bash
   2
   3 # This script executes the script step when running under travis-ci
   4
   5 #if cygwin, check path
   6 case ${MACHTYPE} in
   7         *cygwin*) OPJ_CI_IS_CYGWIN=1;;
   8         *) ;;
   9 esac
  10
  11 # Hack for appveyor to get GNU find in path before windows one.
  12 export PATH=$(dirname ${BASH}):$PATH
  13
  14 # Set-up some bash options
  15 set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
  16 set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
  17 set -o pipefail  ## Fail on error in pipe
  18
  19 function opjpath ()
  20 {
  21         if [ "${OPJ_CI_IS_CYGWIN:-}" == "1" ]; then
  22                 cygpath $1 "$2"
  23         else
  24                 echo "$2"
  25         fi
  26 }
  27
  28 # ABI check is done by abi-check.sh
  29 if [ "${OPJ_CI_ABI_CHECK:-}" == "1" ]; then
  30         exit 0
  31 fi
  32
  33 # Set-up some variables
  34 if [ "${OPJ_CI_BUILD_CONFIGURATION:-}" == "" ]; then
  35         export OPJ_CI_BUILD_CONFIGURATION=Release #default
  36 fi
  37 OPJ_SOURCE_DIR=$(cd $(dirname $0)/../.. && pwd)
  38
  39 if [ "${OPJ_DO_SUBMIT:-}" == "" ]; then
  40         OPJ_DO_SUBMIT=0 # Do not flood cdash by default
  41 fi
  42 if [ "${TRAVIS_REPO_SLUG:-}" != "" ]; then
  43         OPJ_OWNER=$(echo "${TRAVIS_REPO_SLUG}" | sed 's/\(^.*\)\/.*/\1/')
  44         OPJ_SITE="${OPJ_OWNER}.travis-ci.org"
  45         if [ "${OPJ_OWNER}" == "uclouvain" ]; then
  46                 OPJ_DO_SUBMIT=1
  47         fi
  48 elif [ "${APPVEYOR_REPO_NAME:-}" != "" ]; then
  49         OPJ_OWNER=$(echo "${APPVEYOR_REPO_NAME}" | sed 's/\(^.*\)\/.*/\1/')
  50         OPJ_SITE="${OPJ_OWNER}.appveyor.com"
  51         if [ "${OPJ_OWNER}" == "uclouvain" ]; then
  52                 OPJ_DO_SUBMIT=1
  53         fi
  54 else
  55         OPJ_SITE="$(hostname)"
  56 fi
  57
  58 if [ "${TRAVIS_OS_NAME:-}" == "" ]; then
  59   # Let's guess OS for testing purposes
  60         echo "Guessing OS"
  61         if uname -s | grep -i Darwin &> /dev/null; then
  62                 TRAVIS_OS_NAME=osx
  63         elif uname -s | grep -i Linux &> /dev/null; then
  64                 TRAVIS_OS_NAME=linux
  65                 if [ "${CC:-}" == "" ]; then
  66                         # default to gcc
  67                         export CC=gcc
  68                 fi
  69         elif uname -s | grep -i CYGWIN &> /dev/null; then
  70                 TRAVIS_OS_NAME=windows
  71         elif uname -s | grep -i MINGW &> /dev/null; then
  72                 TRAVIS_OS_NAME=windows
  73         elif [ "${APPVEYOR:-}" == "True" ]; then
  74                 TRAVIS_OS_NAME=windows
  75         else
  76                 echo "Failed to guess OS"; exit 1
  77         fi
  78         echo "${TRAVIS_OS_NAME}"
  79 fi
  80
  81 if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
  82         OPJ_OS_NAME=$(sw_vers -productName | tr -d ' ')$(sw_vers -productVersion | sed 's/\([^0-9]*\.[0-9]*\).*/\1/')
  83         OPJ_CC_VERSION=$(xcodebuild -version | grep -i xcode)
  84         OPJ_CC_VERSION=xcode${OPJ_CC_VERSION:6}
  85 elif [ "${TRAVIS_OS_NAME}" == "linux" ]; then
  86         OPJ_OS_NAME=linux
  87         if which lsb_release > /dev/null; then
  88                 OPJ_OS_NAME=$(lsb_release -si)$(lsb_release -sr | sed 's/\([^0-9]*\.[0-9]*\).*/\1/')
  89         fi
  90         if [ -z "${CC##*gcc*}" ]; then
  91                 OPJ_CC_VERSION=$(${CC} --version | head -1 | sed 's/.*\ \([0-9.]*[0-9]\)/\1/')
  92                 if [ -z "${CC##*mingw*}" ]; then
  93                         OPJ_CC_VERSION=mingw${OPJ_CC_VERSION}
  94                         # disable testing for now
  95                         export OPJ_CI_SKIP_TESTS=1
  96                 else
  97                         OPJ_CC_VERSION=gcc${OPJ_CC_VERSION}
  98                 fi
  99         elif [ -z "${CC##*clang*}" ]; then
 100                 OPJ_CC_VERSION=clang$(${CC} --version | grep version | sed 's/.*version \([^0-9.]*[0-9.]*\).*/\1/')
 101         else
 102                 echo "Compiler not supported: ${CC}"; exit 1
 103         fi
 104 elif [ "${TRAVIS_OS_NAME}" == "windows" ]; then
 105         OPJ_OS_NAME=windows
 106         if which cl > /dev/null; then
 107                 OPJ_CL_VERSION=$(cl 2>&1 | grep Version | sed 's/.*Version \([0-9]*\).*/\1/')
 108                 if [ ${OPJ_CL_VERSION} -eq 19 ]; then
 109                         OPJ_CC_VERSION=vs2015
 110                 elif [ ${OPJ_CL_VERSION} -eq 18 ]; then
 111                         OPJ_CC_VERSION=vs2013
 112                 elif [ ${OPJ_CL_VERSION} -eq 17 ]; then
 113                         OPJ_CC_VERSION=vs2012
 114                 elif [ ${OPJ_CL_VERSION} -eq 16 ]; then
 115                         OPJ_CC_VERSION=vs2010
 116                 elif [ ${OPJ_CL_VERSION} -eq 15 ]; then
 117                         OPJ_CC_VERSION=vs2008
 118                 elif [ ${OPJ_CL_VERSION} -eq 14 ]; then
 119                         OPJ_CC_VERSION=vs2005
 120                 else
 121                         OPJ_CC_VERSION=vs????
 122                 fi
 123         fi
 124 else
 125         echo "OS not supported: ${TRAVIS_OS_NAME}"; exit 1
 126 fi
 127
 128 if [ "${OPJ_CI_ARCH:-}" == "" ]; then
 129         echo "Guessing build architecture"
 130         MACHINE_ARCH=$(uname -m)
 131         if [ "${MACHINE_ARCH}" == "x86_64" ]; then
 132                 export OPJ_CI_ARCH=x86_64
 133         fi
 134         echo "${OPJ_CI_ARCH}"
 135 fi
 136
 137 if [ "${TRAVIS_BRANCH:-}" == "" ]; then
 138         if [ "${APPVEYOR_REPO_BRANCH:-}" != "" ]; then
 139                 TRAVIS_BRANCH=${APPVEYOR_REPO_BRANCH}
 140         else
 141                 echo "Guessing branch"
 142                 TRAVIS_BRANCH=$(git -C ${OPJ_SOURCE_DIR} branch | grep '*' | tr -d '*[[:blank:]]')
 143         fi
 144 fi
 145
 146 OPJ_BUILDNAME=${OPJ_OS_NAME}-${OPJ_CC_VERSION}-${OPJ_CI_ARCH}-${TRAVIS_BRANCH}
 147 OPJ_BUILDNAME_TEST=${OPJ_OS_NAME}-${OPJ_CC_VERSION}-${OPJ_CI_ARCH}
 148 if [ "${TRAVIS_PULL_REQUEST:-}" != "false" ] && [ "${TRAVIS_PULL_REQUEST:-}" != "" ]; then
 149         OPJ_BUILDNAME=${OPJ_BUILDNAME}-pr${TRAVIS_PULL_REQUEST}
 150 elif [ "${APPVEYOR_PULL_REQUEST_NUMBER:-}" != "" ]; then
 151         OPJ_BUILDNAME=${OPJ_BUILDNAME}-pr${APPVEYOR_PULL_REQUEST_NUMBER}
 152 fi
 153 OPJ_BUILDNAME=${OPJ_BUILDNAME}-${OPJ_CI_BUILD_CONFIGURATION}-3rdP
 154 OPJ_BUILDNAME_TEST=${OPJ_BUILDNAME_TEST}-${OPJ_CI_BUILD_CONFIGURATION}-3rdP
 155 if [ "${OPJ_CI_ASAN:-}" == "1" ]; then
 156         OPJ_BUILDNAME=${OPJ_BUILDNAME}-ASan
 157         OPJ_BUILDNAME_TEST=${OPJ_BUILDNAME_TEST}-ASan
 158 fi
 159
 160 if [ "${OPJ_NONCOMMERCIAL:-}" == "1" ] && [ "${OPJ_CI_SKIP_TESTS:-}" != "1" ] && [ -d kdu ]; then
 161         echo "
 162 Testing will use Kakadu trial binaries. Here's the copyright notice from kakadu:
 163 Copyright is owned by NewSouth Innovations Pty Limited, commercial arm of the UNSW Australia in Sydney.
 164 You are free to trial these executables and even to re-distribute them,
 165 so long as such use or re-distribution is accompanied with this copyright notice and is not for commercial gain.
 166 Note: Binaries can only be used for non-commercial purposes.
 167 "
 168 fi
 169
 170 if [ -d cmake-install ]; then
 171         export PATH=${PWD}/cmake-install/bin:${PATH}
 172 fi
 173
 174 set -x
 175 # This will print configuration
 176 # travis-ci doesn't dump cmake version in system info, let's print it
 177 cmake --version
 178
 179 export TRAVIS_OS_NAME=${TRAVIS_OS_NAME}
 180 export OPJ_SITE=${OPJ_SITE}
 181 export OPJ_BUILDNAME=${OPJ_BUILDNAME}
 182 export OPJ_SOURCE_DIR=$(opjpath -m ${OPJ_SOURCE_DIR})
 183 export OPJ_BINARY_DIR=$(opjpath -m ${PWD}/build)
 184 export OPJ_BUILD_CONFIGURATION=${OPJ_CI_BUILD_CONFIGURATION}
 185 export OPJ_DO_SUBMIT=${OPJ_DO_SUBMIT}
 186
 187 if [ "${OPJ_SKIP_REBUILD:-}" != "1" ]; then
 188     ctest -S ${OPJ_SOURCE_DIR}/tools/ctest_scripts/travis-ci.cmake -V || true
 189 fi
 190 # ctest will exit with various error codes depending on version.
 191 # ignore ctest exit code & parse this ourselves
 192 set +x
 193
 194 # Deployment if needed
 195 #---------------------
 196 if [ "${TRAVIS_TAG:-}" != "" ]; then
 197                 OPJ_TAG_NAME=${TRAVIS_TAG}
 198         elif [ "${APPVEYOR_REPO_TAG:-}" == "true" ]; then
 199                 OPJ_TAG_NAME=${APPVEYOR_REPO_TAG_NAME}
 200         else
 201                 OPJ_TAG_NAME=""
 202         fi
 203 if [ "${OPJ_CI_INCLUDE_IF_DEPLOY:-}" == "1" ] && [ "${OPJ_TAG_NAME:-}" != "" ]; then
 204 #if [ "${OPJ_CI_INCLUDE_IF_DEPLOY:-}" == "1" ]; then
 205         OPJ_CI_DEPLOY=1         # unused for now
 206         OPJ_CUR_DIR=${PWD}
 207         if [ "${TRAVIS_OS_NAME:-}" == "linux" ]; then
 208                 OPJ_PACK_GENERATOR="TGZ" # ZIP generator currently segfaults on linux
 209         else
 210                 OPJ_PACK_GENERATOR="ZIP"
 211         fi
 212         OPJ_PACK_NAME="openjpeg-${OPJ_TAG_NAME}-${TRAVIS_OS_NAME}-${OPJ_CI_ARCH}"
 213         cd ${OPJ_BINARY_DIR}
 214         cmake -D CPACK_GENERATOR:STRING=${OPJ_PACK_GENERATOR} -D CPACK_PACKAGE_FILE_NAME:STRING=${OPJ_PACK_NAME} ${OPJ_SOURCE_DIR}
 215         cd ${OPJ_CUR_DIR}
 216         cmake --build ${OPJ_BINARY_DIR} --target package
 217         echo "ready to deploy $(ls ${OPJ_BINARY_DIR}/${OPJ_PACK_NAME}*) to GitHub releases"
 218         if [ "${APPVEYOR_REPO_TAG:-}" == "true" ]; then
 219                 appveyor PushArtifact "${OPJ_BINARY_DIR}/${OPJ_PACK_NAME}.zip"
 220         fi
 221 else
 222         OPJ_CI_DEPLOY=0
 223 fi
 224
 225 # let's parse configure/build/tests for failure
 226
 227 echo "
 228 Parsing logs for failures
 229 "
 230 OPJ_CI_RESULT=0
 231
 232 # 1st configure step
 233 OPJ_CONFIGURE_XML=$(find build -path 'build/Testing/*' -name 'Configure.xml')
 234 if [ ! -f "${OPJ_CONFIGURE_XML}" ]; then
 235         echo "No configure log found"
 236         OPJ_CI_RESULT=1
 237 else
 238         if ! grep '<ConfigureStatus>0</ConfigureStatus>' ${OPJ_CONFIGURE_XML} &> /dev/null; then
 239                 echo "Errors were found in configure log"
 240                 OPJ_CI_RESULT=1
 241         fi
 242 fi
 243
 244 # 2nd build step
 245 # We must have one Build.xml file
 246 OPJ_BUILD_XML=$(find build -path 'build/Testing/*' -name 'Build.xml')
 247 if [ ! -f "${OPJ_BUILD_XML}" ]; then
 248         echo "No build log found"
 249         OPJ_CI_RESULT=1
 250 else
 251         if grep '<Error>' ${OPJ_BUILD_XML} &> /dev/null; then
 252                 echo "Errors were found in build log"
 253                 OPJ_CI_RESULT=1
 254         fi
 255 fi
 256
 257 if [ ${OPJ_CI_RESULT} -ne 0 ]; then
 258         # Don't trash output with failing tests when there are configure/build errors
 259         exit ${OPJ_CI_RESULT}
 260 fi
 261
 262 if [ "${OPJ_CI_SKIP_TESTS:-}" != "1" ]; then
 263         OPJ_TEST_XML=$(find build -path 'build/Testing/*' -name 'Test.xml')
 264         if [ ! -f "${OPJ_TEST_XML}" ]; then
 265                 echo "No test log found"
 266                 OPJ_CI_RESULT=1
 267         else
 268                 echo "Parsing tests for new/unknown failures"
 269                 # 3rd test step
 270                 OPJ_FAILEDTEST_LOG=$(find build -path 'build/Testing/Temporary/*' -name 'LastTestsFailed_*.log')
 271                 if [ -f "${OPJ_FAILEDTEST_LOG}" ]; then
 272                         awk -F: '{ print $2 }' ${OPJ_FAILEDTEST_LOG} > failures.txt
 273                         while read FAILEDTEST; do
 274                                 # Start with common errors
 275                                 if grep -x "${FAILEDTEST}" $(opjpath -u ${OPJ_SOURCE_DIR})/tools/travis-ci/knownfailures-all.txt > /dev/null; then
 276                                         continue
 277                                 fi
 278                                 if [ -f $(opjpath -u ${OPJ_SOURCE_DIR})/tools/travis-ci/knownfailures-${OPJ_BUILDNAME_TEST}.txt ]; then
 279                                         if grep -x "${FAILEDTEST}" $(opjpath -u ${OPJ_SOURCE_DIR})/tools/travis-ci/knownfailures-${OPJ_BUILDNAME_TEST}.txt > /dev/null; then
 280                                                 continue
 281                                         fi
 282                                 fi
 283                                 echo "${FAILEDTEST}"
 284                                 OPJ_CI_RESULT=1
 285                         done < failures.txt
 286                 fi
 287         fi
 288
 289         if [ ${OPJ_CI_RESULT} -eq 0 ]; then
 290                 echo "No new/unknown test failure found
 291                 "
 292         else
 293                 echo "
 294 New/unknown test failure found!!!
 295         "
 296         fi
 297
 298         # 4th memcheck step
 299         OPJ_MEMCHECK_XML=$(find build -path 'build/Testing/*' -name 'DynamicAnalysis.xml')
 300         if [ -f "${OPJ_MEMCHECK_XML}" ]; then
 301                 if grep '<Defect Type' ${OPJ_MEMCHECK_XML} 2> /dev/null; then
 302                         echo "Errors were found in dynamic analysis log"
 303                         OPJ_CI_RESULT=1
 304                 fi
 305         fi
 306 fi
 307
 308 if [ "${OPJ_CI_PERF_TESTS:-}" == "1" ]; then
 309     cd tests/performance
 310     echo "Running performance tests on current version (dry-run)"
 311     PATH=../../build/bin:$PATH python ./perf_test.py
 312     echo "Running performance tests on current version"
 313     PATH=../../build/bin:$PATH python ./perf_test.py -o /tmp/new.csv
 314     if [ "${OPJ_NONCOMMERCIAL:-}" == "1" ] && [ -d ../../kdu ]; then
 315         echo "Running performances tests with Kakadu"
 316         LD_LIBRARY_PATH=../../kdu PATH=../../kdu::$PATH python ./perf_test.py -kakadu -o /tmp/kakadu.csv
 317         echo "Comparing current version with Kakadu"
 318         python compare_perfs.py /tmp/kakadu.csv /tmp/new.csv || true
 319     fi
 320     cd ../..
 321
 322     REF_VERSION=master
 323     if [ "${TRAVIS_PULL_REQUEST:-false}" == "false" ]; then
 324         REF_VERSION=v2.1.2
 325     fi
 326     if [ ! -d ref_opj ]; then
 327         git clone https://github.com/uclouvain/openjpeg ref_opj
 328     fi
 329     echo "Building reference version (${REF_VERSION})"
 330     cd ref_opj
 331     git checkout ${REF_VERSION}
 332     mkdir -p build
 333     cd build
 334     cmake .. -DCMAKE_BUILD_TYPE=${OPJ_BUILD_CONFIGURATION}
 335     make -j3
 336     cd ../..
 337     cd tests/performance
 338     echo "Running performance tests on ${REF_VERSION} version (dry-run)"
 339     PATH=../../ref_opj/build/bin:$PATH python ./perf_test.py
 340     echo "Running performance tests on ${REF_VERSION} version"
 341     PATH=../../ref_opj/build/bin:$PATH python ./perf_test.py -o /tmp/ref.csv
 342     echo "Comparing current version with ${REF_VERSION} version"
 343     # we should normally set OPJ_CI_RESULT=1 in case of failure, but
 344     # this is too unreliable
 345     python compare_perfs.py /tmp/ref.csv /tmp/new.csv || true
 346     cd ../..
 347 fi
 348
 349 if [ "${OPJ_CI_PROFILE:-}" == "1" ]; then
 350     rm -rf build_gprof
 351     mkdir build_gprof
 352     cd build_gprof
 353     # We need static linking for gprof
 354     cmake "-DCMAKE_C_FLAGS=-pg -O3" -DCMAKE_EXE_LINKER_FLAGS=-pg -DCMAKE_SHARED_LINKER_FLAGS=-pg -DBUILD_SHARED_LIBS=OFF ..
 355     make -j3
 356     cd ..
 357     build_gprof/bin/opj_decompress -i data/input/nonregression/kodak_2layers_lrcp.j2c -o out.tif > /dev/null
 358     echo "Most CPU consuming functions:"
 359     gprof build_gprof/bin/opj_decompress gmon.out | head || true
 360
 361     rm -f massif.out.*
 362     valgrind --tool=massif build/bin/opj_decompress -i data/input/nonregression/kodak_2layers_lrcp.j2c -o out.tif >/dev/null 2>/dev/null
 363     echo ""
 364     echo "Memory consumption profile:"
 365     python tests/profiling/filter_massif_output.py massif.out.*
 366 fi
 367
 368 exit ${OPJ_CI_RESULT}