Travis error on new failures

6 files changed, 161 insertions, 1 deletions

diff --git a/tools/ctest_scripts/travis-ci.cmake b/tools/ctest_scripts/travis-ci.cmake
index fafc54ee..bab6d7ab 100644
--- a/tools/ctest_scripts/travis-ci.cmake
+++ b/tools/ctest_scripts/travis-ci.cmake
@@ -130,4 +130,5 @@ endif()
 if ("$ENV{OPJ_DO_SUBMIT}" STREQUAL "1")
 	ctest_submit()
 endif()
-ctest_empty_binary_directory( "${CTEST_BINARY_DIRECTORY}" )
+# Do not clean, we'll parse the log for known failure
+#ctest_empty_binary_directory( "${CTEST_BINARY_DIRECTORY}" )
diff --git a/tools/travis-ci/knownfailures-Ubuntu12.04-clang3.4-x86_64-Debug-3rdP-ASan.txt b/tools/travis-ci/knownfailures-Ubuntu12.04-clang3.4-x86_64-Debug-3rdP-ASan.txt
new file mode 100644
index 00000000..ab2169d3
--- /dev/null
+++ b/tools/travis-ci/knownfailures-Ubuntu12.04-clang3.4-x86_64-Debug-3rdP-ASan.txt
@@ -0,0 +1 @@
+NR-ENC-Bretagne2.ppm-7-compare_dec-ref-out2base
diff --git a/tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-i386-Release-3rdP.txt b/tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-i386-Release-3rdP.txt
new file mode 100644
index 00000000..bad45972
--- /dev/null
+++ b/tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-i386-Release-3rdP.txt
@@ -0,0 +1,61 @@
+NR-ENC-issue203-32x32-bgr16.bmp-43-compare_dump2base
+NR-ENC-issue203-32x32-bgr16.bmp-43-compare_dec-ref-out2base
+NR-ENC-issue203-33x33-bgr16.bmp-44-compare_dump2base
+NR-ENC-issue203-33x33-bgr16.bmp-44-compare_dec-ref-out2base
+NR-ENC-issue203-32x32-bgra16.bmp-45-compare_dump2base
+NR-ENC-issue203-33x33-bgra16.bmp-46-compare_dump2base
+NR-ENC-issue203-127x64-bgr16.bmp-49-compare_dump2base
+NR-ENC-issue203-127x64-bgr16.bmp-49-compare_dec-ref-out2base
+NR-DEC-issue104_jpxstream.jp2-33-decode-md5
+NR-DEC-issue226.j2k-74-decode
+NR-DEC-issue226.j2k-74-decode-md5
+NR-DEC-p1_06.j2k-156-decode
+NR-DEC-p1_06.j2k-156-decode-md5
+NR-DEC-p1_06.j2k-164-decode-md5
+NR-C1P0-p0_04.j2k-compare2base
+NR-C1P0-p0_05.j2k-compare2base
+NR-C1P0-p0_06.j2k-compare2base
+NR-C1P1-p1_02.j2k-compare2base
+NR-C1P1-p1_03.j2k-compare2base
+NR-C1P1-p1_04.j2k-compare2base
+NR-C1P1-p1_05.j2k-compare2base
+NR-JP2-file2.jp2-compare2base
+NR-JP2-file3.jp2-compare2base
+NR-RIC-subsampling_1.jp2-compare2base
+NR-RIC-subsampling_2.jp2-compare2base
+NR-RIC-zoo1.jp2-compare2base
+NR-RIC-zoo2.jp2-compare2base
+NR-DEC-_00042.j2k-2-decode-md5
+NR-DEC-buxI.j2k-9-decode-md5
+NR-DEC-CT_Phillips_JPEG2K_Decompr_Problem.j2k-13-decode-md5
+NR-DEC-Marrin.jp2-18-decode-md5
+NR-DEC-kodak_2layers_lrcp.j2c-31-decode-md5
+NR-DEC-kodak_2layers_lrcp.j2c-32-decode-md5
+NR-DEC-file409752.jp2-40-decode-md5
+NR-DEC-issue188_beach_64bitsbox.jp2-41-decode-md5
+NR-DEC-issue206_image-000.jp2-42-decode-md5
+NR-DEC-issue205.jp2-43-decode-md5
+NR-DEC-issue228.j2k-60-decode-md5
+NR-DEC-issue134.jp2-67-decode-md5
+NR-DEC-issue208.jp2-69-decode-md5
+NR-DEC-issue211.jp2-70-decode-md5
+NR-DEC-issue135.j2k-68-decode-md5
+NR-DEC-issue414.jp2-110-decode-md5
+NR-DEC-p1_04.j2k-124-decode-md5
+NR-DEC-p1_04.j2k-125-decode-md5
+NR-DEC-p1_04.j2k-126-decode-md5
+NR-DEC-p1_04.j2k-127-decode-md5
+NR-DEC-p1_04.j2k-128-decode-md5
+NR-DEC-p1_04.j2k-129-decode-md5
+NR-DEC-p1_04.j2k-131-decode-md5
+NR-DEC-p1_04.j2k-134-decode-md5
+NR-DEC-p1_04.j2k-138-decode-md5
+NR-DEC-p1_04.j2k-140-decode-md5
+NR-DEC-p0_04.j2k-166-decode-md5
+NR-DEC-p0_04.j2k-168-decode-md5
+NR-DEC-p0_04.j2k-172-decode-md5
+NR-DEC-issue205.jp2-253-decode-md5
+NR-DEC-issue559-eci-090-CIELab.jp2-255-decode-md5
+NR-DEC-issue236-ESYCC-CDEF.jp2-254-decode-md5
+NR-DEC-issue559-eci-091-CIELab.jp2-256-decode-md5
+NR-DEC-p1_06.j2k-164-decode
diff --git a/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt b/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt
new file mode 100644
index 00000000..fe66aa36
--- /dev/null
+++ b/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt
@@ -0,0 +1,45 @@
+NR-C1P0-p0_04.j2k-compare2base
+NR-C1P0-p0_05.j2k-compare2base
+NR-C1P0-p0_06.j2k-compare2base
+NR-C1P1-p1_02.j2k-compare2base
+NR-C1P1-p1_03.j2k-compare2base
+NR-C1P1-p1_04.j2k-compare2base
+NR-C1P1-p1_05.j2k-compare2base
+NR-JP2-file2.jp2-compare2base
+NR-JP2-file3.jp2-compare2base
+NR-RIC-subsampling_1.jp2-compare2base
+NR-RIC-subsampling_2.jp2-compare2base
+NR-RIC-zoo1.jp2-compare2base
+NR-RIC-zoo2.jp2-compare2base
+NR-DEC-_00042.j2k-2-decode-md5
+NR-DEC-buxI.j2k-9-decode-md5
+NR-DEC-CT_Phillips_JPEG2K_Decompr_Problem.j2k-13-decode-md5
+NR-DEC-Marrin.jp2-18-decode-md5
+NR-DEC-kodak_2layers_lrcp.j2c-31-decode-md5
+NR-DEC-kodak_2layers_lrcp.j2c-32-decode-md5
+NR-DEC-file409752.jp2-40-decode-md5
+NR-DEC-issue188_beach_64bitsbox.jp2-41-decode-md5
+NR-DEC-issue206_image-000.jp2-42-decode-md5
+NR-DEC-issue205.jp2-43-decode-md5
+NR-DEC-issue228.j2k-60-decode-md5
+NR-DEC-issue134.jp2-67-decode-md5
+NR-DEC-issue208.jp2-69-decode-md5
+NR-DEC-issue211.jp2-70-decode-md5
+NR-DEC-issue414.jp2-110-decode-md5
+NR-DEC-p1_04.j2k-124-decode-md5
+NR-DEC-p1_04.j2k-125-decode-md5
+NR-DEC-p1_04.j2k-126-decode-md5
+NR-DEC-p1_04.j2k-127-decode-md5
+NR-DEC-p1_04.j2k-128-decode-md5
+NR-DEC-p1_04.j2k-129-decode-md5
+NR-DEC-p1_04.j2k-131-decode-md5
+NR-DEC-p1_04.j2k-134-decode-md5
+NR-DEC-p1_04.j2k-138-decode-md5
+NR-DEC-p1_04.j2k-140-decode-md5
+NR-DEC-p0_04.j2k-166-decode-md5
+NR-DEC-p0_04.j2k-168-decode-md5
+NR-DEC-p0_04.j2k-172-decode-md5
+NR-DEC-issue205.jp2-253-decode-md5
+NR-DEC-issue236-ESYCC-CDEF.jp2-254-decode-md5
+NR-DEC-issue559-eci-090-CIELab.jp2-255-decode-md5
+NR-DEC-issue559-eci-091-CIELab.jp2-256-decode-md5
diff --git a/tools/travis-ci/knownfailures-all.txt b/tools/travis-ci/knownfailures-all.txt
new file mode 100644
index 00000000..5be40caa
--- /dev/null
+++ b/tools/travis-ci/knownfailures-all.txt
@@ -0,0 +1,17 @@
+NR-DEC-p1_06.j2k-164-decode
+NR-ENC-issue203-32x32-bgr16.bmp-43-compare_dump2base
+NR-ENC-issue203-33x33-bgr16.bmp-44-compare_dump2base
+NR-ENC-issue203-32x32-bgra16.bmp-45-compare_dump2base
+NR-ENC-issue203-33x33-bgra16.bmp-46-compare_dump2base
+NR-ENC-issue203-127x64-bgr16.bmp-49-compare_dump2base
+NR-DEC-issue104_jpxstream.jp2-33-decode-md5
+NR-DEC-issue135.j2k-68-decode-md5
+NR-DEC-issue226.j2k-74-decode-md5
+NR-DEC-p1_06.j2k-156-decode-md5
+NR-DEC-p1_06.j2k-164-decode-md5
+NR-DEC-p1_06.j2k-156-decode
+NR-DEC-issue226.j2k-74-decode
+NR-ENC-issue203-32x32-bgr16.bmp-43-compare_dec-ref-out2base
+NR-ENC-issue203-33x33-bgr16.bmp-44-compare_dec-ref-out2base
+NR-ENC-issue203-127x64-bgr16.bmp-49-compare_dec-ref-out2base
+NR-DEC-broken.jpc-73-decode
diff --git a/tools/travis-ci/run.sh b/tools/travis-ci/run.sh
index 7d056100..c68d2da2 100755
--- a/tools/travis-ci/run.sh
+++ b/tools/travis-ci/run.sh
@@ -85,12 +85,15 @@ if [ "${TRAVIS_BRANCH:-}" == "" ]; then
 fi
 
 OPJ_BUILDNAME=${OPJ_OS_NAME}-${OPJ_CC_VERSION}-${OPJ_CI_ARCH}-${TRAVIS_BRANCH}
+OPJ_BUILDNAME_TEST=${OPJ_OS_NAME}-${OPJ_CC_VERSION}-${OPJ_CI_ARCH}
 if [ "${TRAVIS_PULL_REQUEST:-}" != "false" ] && [ "${TRAVIS_PULL_REQUEST:-}" != "" ]; then
 	OPJ_BUILDNAME=${OPJ_BUILDNAME}-pr${TRAVIS_PULL_REQUEST}
 fi
 OPJ_BUILDNAME=${OPJ_BUILDNAME}-${OPJ_CI_BUILD_CONFIGURATION}-3rdP
+OPJ_BUILDNAME_TEST=${OPJ_BUILDNAME_TEST}-${OPJ_CI_BUILD_CONFIGURATION}-3rdP
 if [ "${OPJ_CI_ASAN:-}" == "1" ]; then
 	OPJ_BUILDNAME=${OPJ_BUILDNAME}-ASan
+	OPJ_BUILDNAME_TEST=${OPJ_BUILDNAME_TEST}-ASan
 fi
 
 if [ "${OPJ_NONCOMMERCIAL:-}" == "1" ] && [ "${OPJ_CI_SKIP_TESTS:-}" != "1" ] && [ -d kdu ]; then
@@ -119,3 +122,35 @@ export OPJ_BUILD_CONFIGURATION=${OPJ_CI_BUILD_CONFIGURATION}
 export OPJ_DO_SUBMIT=${OPJ_DO_SUBMIT}
 
 ctest -S ${OPJ_SOURCE_DIR}/tools/ctest_scripts/travis-ci.cmake -V
+
+# ctest will exit with error code 0 even if tests failed
+# let's parse discarding known failure
+set +x
+echo "
+Parsing logs for new/unknown failures
+"
+OPJ_CI_RESULT=0
+
+awk -F: '{ print $2 }' build/Testing/Temporary/LastTestsFailed_*.log > failures.txt
+
+while read FAILEDTEST; do
+	# Start with common errors
+	if grep -x "${FAILEDTEST}" ${OPJ_SOURCE_DIR}/tools/travis-ci/knownfailures-all.txt > /dev/null; then
+		continue
+	fi
+	if [ -f ${OPJ_SOURCE_DIR}/tools/travis-ci/knownfailures-${OPJ_BUILDNAME_TEST}.txt ]; then
+		if grep -x "${FAILEDTEST}" ${OPJ_SOURCE_DIR}/tools/travis-ci/knownfailures-${OPJ_BUILDNAME_TEST}.txt > /dev/null; then
+			continue
+		fi
+	fi
+	echo "${FAILEDTEST}"
+	OPJ_CI_RESULT=1
+done < failures.txt
+
+# TODO parse memcheck
+
+if [ ${OPJ_CI_RESULT} -eq 0 ]; then
+	echo "No new/unknown failure found"
+fi
+
+exit ${OPJ_CI_RESULT}