From fe4c15f12c562a42a6b0c4b0a0c5e42a25797235 Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Tue, 19 May 2020 18:03:58 +0200 Subject: [PATCH] Testing: revise testing of lossy encoding by comparing PEAK and MSE with original image --- tests/compare_images.c | 54 ++++++++++++++----- tests/nonregression/CMakeLists.txt | 53 +++++++++++++++++- tests/nonregression/test_suite.ctest.in | 16 +++--- ...untu12.04-gcc4.6.3-x86_64-Release-3rdP.txt | 8 --- ...tu14.04-clang3.8.0-x86_64-Release-3rdP.txt | 8 --- ...Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt | 8 --- ...Ubuntu14.04-gcc4.8.4-x86_64-Debug-3rdP.txt | 8 --- 7 files changed, 101 insertions(+), 54 deletions(-) delete mode 100644 tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-x86_64-Release-3rdP.txt delete mode 100644 tools/travis-ci/knownfailures-Ubuntu14.04-clang3.8.0-x86_64-Release-3rdP.txt delete mode 100644 tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-x86_64-Debug-3rdP.txt diff --git a/tests/compare_images.c b/tests/compare_images.c index ed39a1ae..c2d95e5b 100644 --- a/tests/compare_images.c +++ b/tests/compare_images.c @@ -96,6 +96,8 @@ static void compare_images_help_display(void) "for ref/base file and for test file. \n"); fprintf(stdout, " -d \t OPTIONAL \t indicate if you want to run this function as conformance test or as non regression test\n"); + fprintf(stdout, + " -i \t OPTIONAL \t list of features to ignore. Currently 'prec' only supported\n"); fprintf(stdout, "\n"); } @@ -434,6 +436,8 @@ typedef struct test_cmp_parameters { char separator_base[2]; /** */ char separator_test[2]; + /** whether to ignore prec differences */ + int ignore_prec; } test_cmp_parameters; @@ -459,7 +463,8 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param) char *separatorList = NULL; size_t sizemembasefile, sizememtestfile; int index, flagM = 0, flagP = 0; - const char optlist[] = "b:t:n:m:p:s:d"; + const char optlist[] = "b:t:n:m:p:s:di:"; + char* ignoreList = NULL; int c; /* Init parameters*/ @@ -471,6 +476,7 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param) param->nr_flag = 0; param->separator_base[0] = 0; param->separator_test[0] = 0; + param->ignore_prec = 0; opj_opterr = 0; @@ -505,6 +511,9 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param) case 's': separatorList = opj_optarg; break; + case 'i': + ignoreList = opj_optarg; + break; case '?': if ((opj_optopt == 'b') || (opj_optopt == 't') || (opj_optopt == 'n') || (opj_optopt == 'p') || (opj_optopt == 'm') || (opj_optopt @@ -618,6 +627,14 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param) } } + if (ignoreList != NULL) { + if (strcmp(ignoreList, "prec") == 0) { + param->ignore_prec = 1; + } else { + fprintf(stderr, "Unsupported value for -i\n"); + return 1; + } + } if ((param->nr_flag) && (flagP || flagM)) { fprintf(stderr, @@ -645,7 +662,7 @@ int main(int argc, char **argv) char *filenamePNGtest = NULL, *filenamePNGbase = NULL, *filenamePNGdiff = NULL; size_t memsizebasefilename, memsizetestfilename; size_t memsizedifffilename; - int valueDiff = 0, nbPixelDiff = 0; + int nbPixelDiff = 0; double sumDiff = 0.0; /* Structures to store image parameters and data*/ opj_image_t *imageBase = NULL, *imageTest = NULL, *imageDiff = NULL; @@ -790,14 +807,16 @@ int main(int argc, char **argv) goto cleanup; } - if (((imageBase->comps)[it_comp]).prec != ((imageTest->comps)[it_comp]).prec) { + if (((imageBase->comps)[it_comp]).prec != ((imageTest->comps)[it_comp]).prec && + !inParam.ignore_prec) { printf("ERROR: prec mismatch [comp %d] (%d><%d)\n", it_comp, ((imageBase->comps)[it_comp]).prec, ((imageTest->comps)[it_comp]).prec); goto cleanup; } - if (((imageBase->comps)[it_comp]).bpp != ((imageTest->comps)[it_comp]).bpp) { - printf("ERROR: byte per pixel mismatch [comp %d] (%d><%d)\n", it_comp, + if (((imageBase->comps)[it_comp]).bpp != ((imageTest->comps)[it_comp]).bpp && + !inParam.ignore_prec) { + printf("ERROR: bit per pixel mismatch [comp %d] (%d><%d)\n", it_comp, ((imageBase->comps)[it_comp]).bpp, ((imageTest->comps)[it_comp]).bpp); goto cleanup; } @@ -831,16 +850,25 @@ int main(int argc, char **argv) /*printf("filenamePNGdiff = %s [%d / %d octets]\n",filenamePNGdiff, strlen(filenamePNGdiff),memsizedifffilename );*/ /* Compute pixel diff*/ + failed = 0; for (it_comp = 0; it_comp < imageDiff->numcomps; it_comp++) { double SE = 0, PEAK = 0; double MSE = 0; + unsigned right_shift_input = 0; + unsigned right_shift_output = 0; + if (((imageBase->comps)[it_comp]).bpp > ((imageTest->comps)[it_comp]).bpp) { + right_shift_input = ((imageBase->comps)[it_comp]).bpp - (( + imageTest->comps)[it_comp]).bpp; + } else { + right_shift_output = ((imageTest->comps)[it_comp]).bpp - (( + imageBase->comps)[it_comp]).bpp; + } for (itpxl = 0; itpxl < ((imageDiff->comps)[it_comp]).w * ((imageDiff->comps)[it_comp]).h; itpxl++) { - if (abs(((imageBase->comps)[it_comp]).data[itpxl] - (( - imageTest->comps)[it_comp]).data[itpxl]) > 0) { - valueDiff = ((imageBase->comps)[it_comp]).data[itpxl] - (( - imageTest->comps)[it_comp]).data[itpxl]; + int valueDiff = (((imageBase->comps)[it_comp]).data[itpxl] >> right_shift_input) + - (((imageTest->comps)[it_comp]).data[itpxl] >> right_shift_output); + if (valueDiff != 0) { ((imageDiff->comps)[it_comp]).data[itpxl] = abs(valueDiff); sumDiff += valueDiff; nbPixelDiff++; @@ -867,7 +895,7 @@ int main(int argc, char **argv) printf("ERROR: MSE (%f) or PEAK (%f) values produced by the decoded file are greater " "than the allowable error (respectively %f and %f) \n", MSE, PEAK, inParam.tabMSEvalues[it_comp], inParam.tabPEAKvalues[it_comp]); - goto cleanup; + failed = 1; } } else { /* Non regression-test */ if (nbPixelDiff > 0) { @@ -928,13 +956,15 @@ int main(int argc, char **argv) free(filenamePNGdiff_it_comp); } #endif + failed = 1; goto cleanup; } } } /* it_comp loop */ - printf("---- TEST SUCCEED ----\n"); - failed = 0; + if (!failed) { + printf("---- TEST SUCCEED ----\n"); + } cleanup: /*-----------------------------*/ free(param_image_diff); diff --git a/tests/nonregression/CMakeLists.txt b/tests/nonregression/CMakeLists.txt index 9f956b90..f1813ed8 100644 --- a/tests/nonregression/CMakeLists.txt +++ b/tests/nonregression/CMakeLists.txt @@ -248,6 +248,32 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST}) list(REMOVE_AT CMD_ARG_LIST 0) + if(ENC_TEST_FOUND) + + # Parse lines like opj_compress lossy-check { -n 3 -m 0:0:0 -p 0:0:0 } ... + set(LOSSY_CHECK_ARG_LIST "") + list(GET CMD_ARG_LIST 0 NEXT_ARG) + string(REGEX MATCH "^lossy-check$" LOSSY_CHECK ${NEXT_ARG}) + if(LOSSY_CHECK) + list(REMOVE_AT CMD_ARG_LIST 0) + list(GET CMD_ARG_LIST 0 NEXT_ARG) + string(REGEX MATCH "^{$" FOUND_OPEN_CURL ${NEXT_ARG}) + if(NOT FOUND_OPEN_CURL) + message( FATAL_ERROR "'{' expected after lossy-check") + endif() + list(REMOVE_AT CMD_ARG_LIST 0) + while(TRUE) + list(GET CMD_ARG_LIST 0 NEXT_ARG) + list(REMOVE_AT CMD_ARG_LIST 0) + string(REGEX MATCH "^}$" FOUND_CLOSE_CURL ${NEXT_ARG}) + if(FOUND_CLOSE_CURL) + break() + endif() + list (APPEND LOSSY_CHECK_ARG_LIST ${NEXT_ARG}) + endwhile() + endif() + endif() + endif () # Parse the argument list to find the input filename and output filename @@ -320,8 +346,32 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST}) PROPERTIES DEPENDS NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-dump) + if(LOSSY_CHECK) + add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref + COMMAND opj_decompress + -i ${OUTPUT_FILENAME} + -o ${OUTPUT_FILENAME}.tif + ) + + set_tests_properties(NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref + PROPERTIES DEPENDS + NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-encode) + + # Compare the decoding file with original one, using tolerance + add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-compare_dec-ref-out2base + COMMAND compare_images + -b ${INPUT_FILENAME} + -t ${OUTPUT_FILENAME}.tif + -s bXtY + ${LOSSY_CHECK_ARG_LIST} + ) + + set_tests_properties(NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-compare_dec-ref-out2base + PROPERTIES DEPENDS + NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref) + # Decode the encoding file with kakadu expand command - if (KDU_EXPAND_EXECUTABLE) + elseif (KDU_EXPAND_EXECUTABLE) add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref COMMAND ${KDU_EXPAND_EXECUTABLE} -i ${OUTPUT_FILENAME} @@ -344,7 +394,6 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST}) PROPERTIES DEPENDS NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref) endif() - endif() # Test the encoded file is a valid JP2 file diff --git a/tests/nonregression/test_suite.ctest.in b/tests/nonregression/test_suite.ctest.in index 3ed97ab1..3cf4dfc2 100644 --- a/tests/nonregression/test_suite.ctest.in +++ b/tests/nonregression/test_suite.ctest.in @@ -32,16 +32,16 @@ opj_compress -i @INPUT_NR_PATH@/random-issue-0005.tif -o @TEMP_PATH@/random-issu # related to issue 62 opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-u.raw.j2k -F 512,512,1,16,u opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-s.raw.j2k -F 512,512,1,16,s -opj_compress -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_24.j2k -cinema2K 24 -opj_compress -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_24.j2k -cinema2K 24 -opj_compress -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_24.j2k -cinema2K 24 -opj_compress -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_48.j2k -cinema2K 48 -opj_compress -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_48.j2k -cinema2K 48 -opj_compress -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_48.j2k -cinema2K 48 -opj_compress -i @INPUT_NR_PATH@/ElephantDream_4K.tif -o @TEMP_PATH@/ElephantDream_4K_C4K.j2k -cinema4K +opj_compress lossy-check { -n 3 -i prec -m 175:100:212 -p 78:63:91 } -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_24.j2k -cinema2K 24 +opj_compress lossy-check { -n 3 -i prec -m 298:168:363 -p 121:73:164 } -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_24.j2k -cinema2K 24 +opj_compress lossy-check { -n 3 -i prec -m 76:54:140 -p 55:49:74 } -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_24.j2k -cinema2K 24 +opj_compress lossy-check { -n 3 -i prec -m 384:385:842 -p 134:146:200 } -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_48.j2k -cinema2K 48 +opj_compress lossy-check { -n 3 -i prec -m 933:827:2206 -p 201:184:314 } -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_48.j2k -cinema2K 48 +opj_compress lossy-check { -n 3 -i prec -m 194:173:531 -p 94:79:154 } -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_48.j2k -cinema2K 48 +opj_compress lossy-check { -n 3 -i prec -m 6:4:7 -p 141:141:193 } -i @INPUT_NR_PATH@/ElephantDream_4K.tif -o @TEMP_PATH@/ElephantDream_4K_C4K.j2k -cinema4K # issue 141 opj_compress -i @INPUT_NR_PATH@/issue141.rawl -o @TEMP_PATH@/issue141.rawl.j2k -F 2048,32,1,16,u -opj_compress -i @INPUT_NR_PATH@/issue141.rawl -o @TEMP_PATH@/issue141-I.rawl.j2k -F 2048,32,1,16,u -I +opj_compress lossy-check { -n 1 -m 61 -p 11 } -i @INPUT_NR_PATH@/issue141.tif -o @TEMP_PATH@/issue141-I.rawl.j2k -I # issue 46: opj_compress -i @INPUT_NR_PATH@/Bretagne2.ppm -o @TEMP_PATH@/Bretagne2_5.j2k -c [64,64] # issue 316 diff --git a/tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-x86_64-Release-3rdP.txt b/tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-x86_64-Release-3rdP.txt deleted file mode 100644 index 2554d91c..00000000 --- a/tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-x86_64-Release-3rdP.txt +++ /dev/null @@ -1,8 +0,0 @@ -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base -NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base -NR-ENC-issue141.rawl-23-compare_dec-ref-out2base diff --git a/tools/travis-ci/knownfailures-Ubuntu14.04-clang3.8.0-x86_64-Release-3rdP.txt b/tools/travis-ci/knownfailures-Ubuntu14.04-clang3.8.0-x86_64-Release-3rdP.txt deleted file mode 100644 index 2554d91c..00000000 --- a/tools/travis-ci/knownfailures-Ubuntu14.04-clang3.8.0-x86_64-Release-3rdP.txt +++ /dev/null @@ -1,8 +0,0 @@ -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base -NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base -NR-ENC-issue141.rawl-23-compare_dec-ref-out2base diff --git a/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt b/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt index 9615008a..8e53a623 100644 --- a/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt +++ b/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt @@ -46,11 +46,3 @@ NR-DEC-kodak_2layers_lrcp.j2c-32-decode-md5 NR-DEC-issue135.j2k-68-decode-md5 NR-DEC-db11217111510058.jp2-306-decode-md5 NR-DEC-tnsot_zero.jp2-307-decode-md5 -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base -NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base -NR-ENC-issue141.rawl-23-compare_dec-ref-out2base diff --git a/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-x86_64-Debug-3rdP.txt b/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-x86_64-Debug-3rdP.txt deleted file mode 100644 index 2554d91c..00000000 --- a/tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-x86_64-Debug-3rdP.txt +++ /dev/null @@ -1,8 +0,0 @@ -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base -NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base -NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base -NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base -NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base -NR-ENC-issue141.rawl-23-compare_dec-ref-out2base