Testing: revise testing of lossy encoding by comparing PEAK and MSE with original image

tests/compare_images.c

@@ -96,6 +96,8 @@ static void compare_images_help_display(void)
             "for ref/base file and for test file.  \n");
     fprintf(stdout,
             "  -d \t OPTIONAL \t indicate if you want to run this function as conformance test or as non regression test\n");
+    fprintf(stdout,
+            "  -i \t OPTIONAL \t list of features to ignore. Currently 'prec' only supported\n");
     fprintf(stdout, "\n");
 }
 
@@ -434,6 +436,8 @@ typedef struct test_cmp_parameters {
     char separator_base[2];
     /**  */
     char separator_test[2];
+    /** whether to ignore prec differences */
+    int ignore_prec;
 
 } test_cmp_parameters;
 
@@ -459,7 +463,8 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
     char *separatorList = NULL;
     size_t sizemembasefile, sizememtestfile;
     int index, flagM = 0, flagP = 0;
-    const char optlist[] = "b:t:n:m:p:s:d";
+    const char optlist[] = "b:t:n:m:p:s:di:";
+    char* ignoreList = NULL;
     int c;
 
     /* Init parameters*/
@@ -471,6 +476,7 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
     param->nr_flag = 0;
     param->separator_base[0] = 0;
     param->separator_test[0] = 0;
+    param->ignore_prec = 0;
 
     opj_opterr = 0;
 
@@ -505,6 +511,9 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
         case 's':
             separatorList = opj_optarg;
             break;
+        case 'i':
+            ignoreList = opj_optarg;
+            break;
         case '?':
             if ((opj_optopt == 'b') || (opj_optopt == 't') || (opj_optopt == 'n') ||
                     (opj_optopt == 'p') || (opj_optopt == 'm') || (opj_optopt
@@ -618,6 +627,14 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
         }
     }
 
+    if (ignoreList != NULL) {
+        if (strcmp(ignoreList, "prec") == 0) {
+            param->ignore_prec = 1;
+        } else {
+            fprintf(stderr, "Unsupported value for -i\n");
+            return 1;
+        }
+    }
 
     if ((param->nr_flag) && (flagP || flagM)) {
         fprintf(stderr,
@@ -645,7 +662,7 @@ int main(int argc, char **argv)
     char *filenamePNGtest = NULL, *filenamePNGbase = NULL, *filenamePNGdiff = NULL;
     size_t memsizebasefilename, memsizetestfilename;
     size_t memsizedifffilename;
-    int valueDiff = 0, nbPixelDiff = 0;
+    int nbPixelDiff = 0;
     double sumDiff = 0.0;
     /* Structures to store image parameters and data*/
     opj_image_t *imageBase = NULL, *imageTest = NULL, *imageDiff = NULL;
@@ -790,14 +807,16 @@ int main(int argc, char **argv)
             goto cleanup;
         }
 
-        if (((imageBase->comps)[it_comp]).prec != ((imageTest->comps)[it_comp]).prec) {
+        if (((imageBase->comps)[it_comp]).prec != ((imageTest->comps)[it_comp]).prec &&
+                !inParam.ignore_prec) {
             printf("ERROR: prec mismatch [comp %d] (%d><%d)\n", it_comp,
                    ((imageBase->comps)[it_comp]).prec, ((imageTest->comps)[it_comp]).prec);
             goto cleanup;
         }
 
-        if (((imageBase->comps)[it_comp]).bpp != ((imageTest->comps)[it_comp]).bpp) {
+        if (((imageBase->comps)[it_comp]).bpp != ((imageTest->comps)[it_comp]).bpp &&
-            printf("ERROR: byte per pixel mismatch [comp %d] (%d><%d)\n", it_comp,
+                !inParam.ignore_prec) {
+            printf("ERROR: bit per pixel mismatch [comp %d] (%d><%d)\n", it_comp,
                    ((imageBase->comps)[it_comp]).bpp, ((imageTest->comps)[it_comp]).bpp);
             goto cleanup;
         }
@@ -831,16 +850,25 @@ int main(int argc, char **argv)
     /*printf("filenamePNGdiff = %s [%d / %d octets]\n",filenamePNGdiff, strlen(filenamePNGdiff),memsizedifffilename );*/
 
     /* Compute pixel diff*/
+    failed = 0;
     for (it_comp = 0; it_comp < imageDiff->numcomps; it_comp++) {
         double SE = 0, PEAK = 0;
         double MSE = 0;
+        unsigned right_shift_input = 0;
+        unsigned right_shift_output = 0;
+        if (((imageBase->comps)[it_comp]).bpp > ((imageTest->comps)[it_comp]).bpp) {
+            right_shift_input = ((imageBase->comps)[it_comp]).bpp - ((
+                                    imageTest->comps)[it_comp]).bpp;
+        } else {
+            right_shift_output = ((imageTest->comps)[it_comp]).bpp - ((
+                                     imageBase->comps)[it_comp]).bpp;
+        }
         for (itpxl = 0;
                 itpxl < ((imageDiff->comps)[it_comp]).w * ((imageDiff->comps)[it_comp]).h;
                 itpxl++) {
-            if (abs(((imageBase->comps)[it_comp]).data[itpxl] - ((
+            int valueDiff = (((imageBase->comps)[it_comp]).data[itpxl] >> right_shift_input)
-                        imageTest->comps)[it_comp]).data[itpxl]) > 0) {
+                            - (((imageTest->comps)[it_comp]).data[itpxl] >> right_shift_output);
-                valueDiff = ((imageBase->comps)[it_comp]).data[itpxl] - ((
+            if (valueDiff != 0) {
-                                imageTest->comps)[it_comp]).data[itpxl];
                 ((imageDiff->comps)[it_comp]).data[itpxl] = abs(valueDiff);
                 sumDiff += valueDiff;
                 nbPixelDiff++;
@@ -867,7 +895,7 @@ int main(int argc, char **argv)
                 printf("ERROR: MSE (%f) or PEAK (%f) values produced by the decoded file are greater "
                        "than the allowable error (respectively %f and %f) \n",
                        MSE, PEAK, inParam.tabMSEvalues[it_comp], inParam.tabPEAKvalues[it_comp]);
-                goto cleanup;
+                failed = 1;
             }
         } else { /* Non regression-test */
             if (nbPixelDiff > 0) {
@@ -928,13 +956,15 @@ int main(int argc, char **argv)
                     free(filenamePNGdiff_it_comp);
                 }
 #endif
+                failed = 1;
                 goto cleanup;
             }
         }
     } /* it_comp loop */
 
-    printf("---- TEST SUCCEED ----\n");
+    if (!failed) {
-    failed = 0;
+        printf("---- TEST SUCCEED ----\n");
+    }
 cleanup:
     /*-----------------------------*/
     free(param_image_diff);

tests/nonregression/CMakeLists.txt

@@ -248,6 +248,32 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST})
 
       list(REMOVE_AT CMD_ARG_LIST 0)
 
+      if(ENC_TEST_FOUND)
+
+        # Parse lines like opj_compress lossy-check { -n 3 -m 0:0:0 -p 0:0:0 } ...
+        set(LOSSY_CHECK_ARG_LIST "")
+        list(GET CMD_ARG_LIST 0 NEXT_ARG)
+        string(REGEX MATCH "^lossy-check$" LOSSY_CHECK ${NEXT_ARG})
+        if(LOSSY_CHECK)
+            list(REMOVE_AT CMD_ARG_LIST 0)
+            list(GET CMD_ARG_LIST 0 NEXT_ARG)
+            string(REGEX MATCH "^{$" FOUND_OPEN_CURL ${NEXT_ARG})
+            if(NOT FOUND_OPEN_CURL)
+                message( FATAL_ERROR "'{' expected after lossy-check")
+            endif()
+            list(REMOVE_AT CMD_ARG_LIST 0)
+            while(TRUE)
+                list(GET CMD_ARG_LIST 0 NEXT_ARG)
+                list(REMOVE_AT CMD_ARG_LIST 0)
+                string(REGEX MATCH "^}$" FOUND_CLOSE_CURL ${NEXT_ARG})
+                if(FOUND_CLOSE_CURL)
+                    break()
+                endif()
+                list (APPEND LOSSY_CHECK_ARG_LIST ${NEXT_ARG})
+            endwhile()
+        endif()
+      endif()
+
     endif ()
 
     # Parse the argument list to find the input filename and output filename
@@ -320,8 +346,32 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST})
                                PROPERTIES DEPENDS
                                NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-dump)
 
+          if(LOSSY_CHECK)
+            add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref
+            COMMAND opj_decompress
+            -i ${OUTPUT_FILENAME}
+            -o ${OUTPUT_FILENAME}.tif
+            )
+
+            set_tests_properties(NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref
+                                PROPERTIES DEPENDS
+                                NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-encode)
+
+            # Compare the decoding file with original one, using tolerance
+            add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-compare_dec-ref-out2base
+            COMMAND compare_images
+            -b ${INPUT_FILENAME}
+            -t ${OUTPUT_FILENAME}.tif
+            -s bXtY
+            ${LOSSY_CHECK_ARG_LIST}
+            )
+
+            set_tests_properties(NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-compare_dec-ref-out2base
+                                PROPERTIES DEPENDS
+                                NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref)
+
           # Decode the encoding file with kakadu expand command
-          if (KDU_EXPAND_EXECUTABLE)
+          elseif (KDU_EXPAND_EXECUTABLE)
             add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref
               COMMAND ${KDU_EXPAND_EXECUTABLE}
               -i ${OUTPUT_FILENAME}
@@ -344,7 +394,6 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST})
                                     PROPERTIES DEPENDS
                                     NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref)
             endif()
-
           endif()
 
           # Test the encoded file is a valid JP2 file

tests/nonregression/test_suite.ctest.in

@@ -32,16 +32,16 @@ opj_compress -i @INPUT_NR_PATH@/random-issue-0005.tif -o @TEMP_PATH@/random-issu
 # related to issue 62
 opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-u.raw.j2k -F 512,512,1,16,u
 opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-s.raw.j2k -F 512,512,1,16,s
-opj_compress -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_24.j2k -cinema2K 24
+opj_compress lossy-check { -n 3 -i prec -m 175:100:212 -p 78:63:91 } -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_24.j2k -cinema2K 24
-opj_compress -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_24.j2k -cinema2K 24
+opj_compress lossy-check { -n 3 -i prec -m 298:168:363 -p 121:73:164 } -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_24.j2k -cinema2K 24
-opj_compress -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_24.j2k -cinema2K 24
+opj_compress lossy-check { -n 3 -i prec -m 76:54:140 -p 55:49:74 } -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_24.j2k -cinema2K 24
-opj_compress -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_48.j2k -cinema2K 48
+opj_compress lossy-check { -n 3 -i prec -m 384:385:842 -p 134:146:200 } -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_48.j2k -cinema2K 48
-opj_compress -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_48.j2k -cinema2K 48
+opj_compress lossy-check { -n 3 -i prec -m 933:827:2206 -p 201:184:314 } -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_48.j2k -cinema2K 48
-opj_compress -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_48.j2k -cinema2K 48
+opj_compress lossy-check { -n 3 -i prec -m 194:173:531 -p 94:79:154 } -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_48.j2k -cinema2K 48
-opj_compress -i @INPUT_NR_PATH@/ElephantDream_4K.tif -o @TEMP_PATH@/ElephantDream_4K_C4K.j2k -cinema4K
+opj_compress lossy-check { -n 3 -i prec -m 6:4:7 -p 141:141:193 } -i @INPUT_NR_PATH@/ElephantDream_4K.tif -o @TEMP_PATH@/ElephantDream_4K_C4K.j2k -cinema4K
 # issue 141
 opj_compress -i @INPUT_NR_PATH@/issue141.rawl -o @TEMP_PATH@/issue141.rawl.j2k   -F 2048,32,1,16,u
-opj_compress -i @INPUT_NR_PATH@/issue141.rawl -o @TEMP_PATH@/issue141-I.rawl.j2k -F 2048,32,1,16,u -I
+opj_compress lossy-check { -n 1 -m 61 -p 11 } -i @INPUT_NR_PATH@/issue141.tif -o @TEMP_PATH@/issue141-I.rawl.j2k -I
 # issue 46:
 opj_compress -i @INPUT_NR_PATH@/Bretagne2.ppm -o @TEMP_PATH@/Bretagne2_5.j2k -c [64,64]
 # issue 316

tools/travis-ci/knownfailures-Ubuntu12.04-gcc4.6.3-x86_64-Release-3rdP.txt

@@ -1,8 +0,0 @@
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
-NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
-NR-ENC-issue141.rawl-23-compare_dec-ref-out2base

tools/travis-ci/knownfailures-Ubuntu14.04-clang3.8.0-x86_64-Release-3rdP.txt

@@ -1,8 +0,0 @@
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
-NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
-NR-ENC-issue141.rawl-23-compare_dec-ref-out2base

tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-i386-Release-3rdP.txt

@@ -46,11 +46,3 @@ NR-DEC-kodak_2layers_lrcp.j2c-32-decode-md5
 NR-DEC-issue135.j2k-68-decode-md5
 NR-DEC-db11217111510058.jp2-306-decode-md5
 NR-DEC-tnsot_zero.jp2-307-decode-md5
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
-NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
-NR-ENC-issue141.rawl-23-compare_dec-ref-out2base

tools/travis-ci/knownfailures-Ubuntu14.04-gcc4.8.4-x86_64-Debug-3rdP.txt

@@ -1,8 +0,0 @@
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
-NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
-NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
-NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
-NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
-NR-ENC-issue141.rawl-23-compare_dec-ref-out2base