Testing: revise testing of lossy encoding by comparing PEAK and MSE with original image

This commit is contained in:
Even Rouault 2020-05-19 18:03:58 +02:00
parent c2b9d09c65
commit fe4c15f12c
No known key found for this signature in database
GPG Key ID: 33EBBFC47B3DD87D
7 changed files with 101 additions and 54 deletions

View File

@ -96,6 +96,8 @@ static void compare_images_help_display(void)
"for ref/base file and for test file. \n"); "for ref/base file and for test file. \n");
fprintf(stdout, fprintf(stdout,
" -d \t OPTIONAL \t indicate if you want to run this function as conformance test or as non regression test\n"); " -d \t OPTIONAL \t indicate if you want to run this function as conformance test or as non regression test\n");
fprintf(stdout,
" -i \t OPTIONAL \t list of features to ignore. Currently 'prec' only supported\n");
fprintf(stdout, "\n"); fprintf(stdout, "\n");
} }
@ -434,6 +436,8 @@ typedef struct test_cmp_parameters {
char separator_base[2]; char separator_base[2];
/** */ /** */
char separator_test[2]; char separator_test[2];
/** whether to ignore prec differences */
int ignore_prec;
} test_cmp_parameters; } test_cmp_parameters;
@ -459,7 +463,8 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
char *separatorList = NULL; char *separatorList = NULL;
size_t sizemembasefile, sizememtestfile; size_t sizemembasefile, sizememtestfile;
int index, flagM = 0, flagP = 0; int index, flagM = 0, flagP = 0;
const char optlist[] = "b:t:n:m:p:s:d"; const char optlist[] = "b:t:n:m:p:s:di:";
char* ignoreList = NULL;
int c; int c;
/* Init parameters*/ /* Init parameters*/
@ -471,6 +476,7 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
param->nr_flag = 0; param->nr_flag = 0;
param->separator_base[0] = 0; param->separator_base[0] = 0;
param->separator_test[0] = 0; param->separator_test[0] = 0;
param->ignore_prec = 0;
opj_opterr = 0; opj_opterr = 0;
@ -505,6 +511,9 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
case 's': case 's':
separatorList = opj_optarg; separatorList = opj_optarg;
break; break;
case 'i':
ignoreList = opj_optarg;
break;
case '?': case '?':
if ((opj_optopt == 'b') || (opj_optopt == 't') || (opj_optopt == 'n') || if ((opj_optopt == 'b') || (opj_optopt == 't') || (opj_optopt == 'n') ||
(opj_optopt == 'p') || (opj_optopt == 'm') || (opj_optopt (opj_optopt == 'p') || (opj_optopt == 'm') || (opj_optopt
@ -618,6 +627,14 @@ static int parse_cmdline_cmp(int argc, char **argv, test_cmp_parameters* param)
} }
} }
if (ignoreList != NULL) {
if (strcmp(ignoreList, "prec") == 0) {
param->ignore_prec = 1;
} else {
fprintf(stderr, "Unsupported value for -i\n");
return 1;
}
}
if ((param->nr_flag) && (flagP || flagM)) { if ((param->nr_flag) && (flagP || flagM)) {
fprintf(stderr, fprintf(stderr,
@ -645,7 +662,7 @@ int main(int argc, char **argv)
char *filenamePNGtest = NULL, *filenamePNGbase = NULL, *filenamePNGdiff = NULL; char *filenamePNGtest = NULL, *filenamePNGbase = NULL, *filenamePNGdiff = NULL;
size_t memsizebasefilename, memsizetestfilename; size_t memsizebasefilename, memsizetestfilename;
size_t memsizedifffilename; size_t memsizedifffilename;
int valueDiff = 0, nbPixelDiff = 0; int nbPixelDiff = 0;
double sumDiff = 0.0; double sumDiff = 0.0;
/* Structures to store image parameters and data*/ /* Structures to store image parameters and data*/
opj_image_t *imageBase = NULL, *imageTest = NULL, *imageDiff = NULL; opj_image_t *imageBase = NULL, *imageTest = NULL, *imageDiff = NULL;
@ -790,14 +807,16 @@ int main(int argc, char **argv)
goto cleanup; goto cleanup;
} }
if (((imageBase->comps)[it_comp]).prec != ((imageTest->comps)[it_comp]).prec) { if (((imageBase->comps)[it_comp]).prec != ((imageTest->comps)[it_comp]).prec &&
!inParam.ignore_prec) {
printf("ERROR: prec mismatch [comp %d] (%d><%d)\n", it_comp, printf("ERROR: prec mismatch [comp %d] (%d><%d)\n", it_comp,
((imageBase->comps)[it_comp]).prec, ((imageTest->comps)[it_comp]).prec); ((imageBase->comps)[it_comp]).prec, ((imageTest->comps)[it_comp]).prec);
goto cleanup; goto cleanup;
} }
if (((imageBase->comps)[it_comp]).bpp != ((imageTest->comps)[it_comp]).bpp) { if (((imageBase->comps)[it_comp]).bpp != ((imageTest->comps)[it_comp]).bpp &&
printf("ERROR: byte per pixel mismatch [comp %d] (%d><%d)\n", it_comp, !inParam.ignore_prec) {
printf("ERROR: bit per pixel mismatch [comp %d] (%d><%d)\n", it_comp,
((imageBase->comps)[it_comp]).bpp, ((imageTest->comps)[it_comp]).bpp); ((imageBase->comps)[it_comp]).bpp, ((imageTest->comps)[it_comp]).bpp);
goto cleanup; goto cleanup;
} }
@ -831,16 +850,25 @@ int main(int argc, char **argv)
/*printf("filenamePNGdiff = %s [%d / %d octets]\n",filenamePNGdiff, strlen(filenamePNGdiff),memsizedifffilename );*/ /*printf("filenamePNGdiff = %s [%d / %d octets]\n",filenamePNGdiff, strlen(filenamePNGdiff),memsizedifffilename );*/
/* Compute pixel diff*/ /* Compute pixel diff*/
failed = 0;
for (it_comp = 0; it_comp < imageDiff->numcomps; it_comp++) { for (it_comp = 0; it_comp < imageDiff->numcomps; it_comp++) {
double SE = 0, PEAK = 0; double SE = 0, PEAK = 0;
double MSE = 0; double MSE = 0;
unsigned right_shift_input = 0;
unsigned right_shift_output = 0;
if (((imageBase->comps)[it_comp]).bpp > ((imageTest->comps)[it_comp]).bpp) {
right_shift_input = ((imageBase->comps)[it_comp]).bpp - ((
imageTest->comps)[it_comp]).bpp;
} else {
right_shift_output = ((imageTest->comps)[it_comp]).bpp - ((
imageBase->comps)[it_comp]).bpp;
}
for (itpxl = 0; for (itpxl = 0;
itpxl < ((imageDiff->comps)[it_comp]).w * ((imageDiff->comps)[it_comp]).h; itpxl < ((imageDiff->comps)[it_comp]).w * ((imageDiff->comps)[it_comp]).h;
itpxl++) { itpxl++) {
if (abs(((imageBase->comps)[it_comp]).data[itpxl] - (( int valueDiff = (((imageBase->comps)[it_comp]).data[itpxl] >> right_shift_input)
imageTest->comps)[it_comp]).data[itpxl]) > 0) { - (((imageTest->comps)[it_comp]).data[itpxl] >> right_shift_output);
valueDiff = ((imageBase->comps)[it_comp]).data[itpxl] - (( if (valueDiff != 0) {
imageTest->comps)[it_comp]).data[itpxl];
((imageDiff->comps)[it_comp]).data[itpxl] = abs(valueDiff); ((imageDiff->comps)[it_comp]).data[itpxl] = abs(valueDiff);
sumDiff += valueDiff; sumDiff += valueDiff;
nbPixelDiff++; nbPixelDiff++;
@ -867,7 +895,7 @@ int main(int argc, char **argv)
printf("ERROR: MSE (%f) or PEAK (%f) values produced by the decoded file are greater " printf("ERROR: MSE (%f) or PEAK (%f) values produced by the decoded file are greater "
"than the allowable error (respectively %f and %f) \n", "than the allowable error (respectively %f and %f) \n",
MSE, PEAK, inParam.tabMSEvalues[it_comp], inParam.tabPEAKvalues[it_comp]); MSE, PEAK, inParam.tabMSEvalues[it_comp], inParam.tabPEAKvalues[it_comp]);
goto cleanup; failed = 1;
} }
} else { /* Non regression-test */ } else { /* Non regression-test */
if (nbPixelDiff > 0) { if (nbPixelDiff > 0) {
@ -928,13 +956,15 @@ int main(int argc, char **argv)
free(filenamePNGdiff_it_comp); free(filenamePNGdiff_it_comp);
} }
#endif #endif
failed = 1;
goto cleanup; goto cleanup;
} }
} }
} /* it_comp loop */ } /* it_comp loop */
printf("---- TEST SUCCEED ----\n"); if (!failed) {
failed = 0; printf("---- TEST SUCCEED ----\n");
}
cleanup: cleanup:
/*-----------------------------*/ /*-----------------------------*/
free(param_image_diff); free(param_image_diff);

View File

@ -248,6 +248,32 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST})
list(REMOVE_AT CMD_ARG_LIST 0) list(REMOVE_AT CMD_ARG_LIST 0)
if(ENC_TEST_FOUND)
# Parse lines like opj_compress lossy-check { -n 3 -m 0:0:0 -p 0:0:0 } ...
set(LOSSY_CHECK_ARG_LIST "")
list(GET CMD_ARG_LIST 0 NEXT_ARG)
string(REGEX MATCH "^lossy-check$" LOSSY_CHECK ${NEXT_ARG})
if(LOSSY_CHECK)
list(REMOVE_AT CMD_ARG_LIST 0)
list(GET CMD_ARG_LIST 0 NEXT_ARG)
string(REGEX MATCH "^{$" FOUND_OPEN_CURL ${NEXT_ARG})
if(NOT FOUND_OPEN_CURL)
message( FATAL_ERROR "'{' expected after lossy-check")
endif()
list(REMOVE_AT CMD_ARG_LIST 0)
while(TRUE)
list(GET CMD_ARG_LIST 0 NEXT_ARG)
list(REMOVE_AT CMD_ARG_LIST 0)
string(REGEX MATCH "^}$" FOUND_CLOSE_CURL ${NEXT_ARG})
if(FOUND_CLOSE_CURL)
break()
endif()
list (APPEND LOSSY_CHECK_ARG_LIST ${NEXT_ARG})
endwhile()
endif()
endif()
endif () endif ()
# Parse the argument list to find the input filename and output filename # Parse the argument list to find the input filename and output filename
@ -320,8 +346,32 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST})
PROPERTIES DEPENDS PROPERTIES DEPENDS
NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-dump) NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-dump)
if(LOSSY_CHECK)
add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref
COMMAND opj_decompress
-i ${OUTPUT_FILENAME}
-o ${OUTPUT_FILENAME}.tif
)
set_tests_properties(NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref
PROPERTIES DEPENDS
NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-encode)
# Compare the decoding file with original one, using tolerance
add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-compare_dec-ref-out2base
COMMAND compare_images
-b ${INPUT_FILENAME}
-t ${OUTPUT_FILENAME}.tif
-s bXtY
${LOSSY_CHECK_ARG_LIST}
)
set_tests_properties(NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-compare_dec-ref-out2base
PROPERTIES DEPENDS
NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref)
# Decode the encoding file with kakadu expand command # Decode the encoding file with kakadu expand command
if (KDU_EXPAND_EXECUTABLE) elseif (KDU_EXPAND_EXECUTABLE)
add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref add_test(NAME NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref
COMMAND ${KDU_EXPAND_EXECUTABLE} COMMAND ${KDU_EXPAND_EXECUTABLE}
-i ${OUTPUT_FILENAME} -i ${OUTPUT_FILENAME}
@ -344,7 +394,6 @@ foreach(OPJ_TEST_CMD_LINE ${OPJ_TEST_CMD_LINE_LIST})
PROPERTIES DEPENDS PROPERTIES DEPENDS
NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref) NR-ENC-${INPUT_FILENAME_NAME}-${IT_TEST_ENC}-decode-ref)
endif() endif()
endif() endif()
# Test the encoded file is a valid JP2 file # Test the encoded file is a valid JP2 file

View File

@ -32,16 +32,16 @@ opj_compress -i @INPUT_NR_PATH@/random-issue-0005.tif -o @TEMP_PATH@/random-issu
# related to issue 62 # related to issue 62
opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-u.raw.j2k -F 512,512,1,16,u opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-u.raw.j2k -F 512,512,1,16,u
opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-s.raw.j2k -F 512,512,1,16,s opj_compress -i @INPUT_NR_PATH@/tmp-issue-0062.raw -o @TEMP_PATH@/tmp-issue-0062-s.raw.j2k -F 512,512,1,16,s
opj_compress -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_24.j2k -cinema2K 24 opj_compress lossy-check { -n 3 -i prec -m 175:100:212 -p 78:63:91 } -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_24.j2k -cinema2K 24
opj_compress -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_24.j2k -cinema2K 24 opj_compress lossy-check { -n 3 -i prec -m 298:168:363 -p 121:73:164 } -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_24.j2k -cinema2K 24
opj_compress -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_24.j2k -cinema2K 24 opj_compress lossy-check { -n 3 -i prec -m 76:54:140 -p 55:49:74 } -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_24.j2k -cinema2K 24
opj_compress -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_48.j2k -cinema2K 48 opj_compress lossy-check { -n 3 -i prec -m 384:385:842 -p 134:146:200 } -i @INPUT_NR_PATH@/X_4_2K_24_185_CBR_WB_000.tif -o @TEMP_PATH@/X_4_2K_24_185_CBR_WB_000_C2K_48.j2k -cinema2K 48
opj_compress -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_48.j2k -cinema2K 48 opj_compress lossy-check { -n 3 -i prec -m 933:827:2206 -p 201:184:314 } -i @INPUT_NR_PATH@/X_5_2K_24_235_CBR_STEM24_000.tif -o @TEMP_PATH@/X_5_2K_24_235_CBR_STEM24_000_C2K_48.j2k -cinema2K 48
opj_compress -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_48.j2k -cinema2K 48 opj_compress lossy-check { -n 3 -i prec -m 194:173:531 -p 94:79:154 } -i @INPUT_NR_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000.tif -o @TEMP_PATH@/X_6_2K_24_FULL_CBR_CIRCLE_000_C2K_48.j2k -cinema2K 48
opj_compress -i @INPUT_NR_PATH@/ElephantDream_4K.tif -o @TEMP_PATH@/ElephantDream_4K_C4K.j2k -cinema4K opj_compress lossy-check { -n 3 -i prec -m 6:4:7 -p 141:141:193 } -i @INPUT_NR_PATH@/ElephantDream_4K.tif -o @TEMP_PATH@/ElephantDream_4K_C4K.j2k -cinema4K
# issue 141 # issue 141
opj_compress -i @INPUT_NR_PATH@/issue141.rawl -o @TEMP_PATH@/issue141.rawl.j2k -F 2048,32,1,16,u opj_compress -i @INPUT_NR_PATH@/issue141.rawl -o @TEMP_PATH@/issue141.rawl.j2k -F 2048,32,1,16,u
opj_compress -i @INPUT_NR_PATH@/issue141.rawl -o @TEMP_PATH@/issue141-I.rawl.j2k -F 2048,32,1,16,u -I opj_compress lossy-check { -n 1 -m 61 -p 11 } -i @INPUT_NR_PATH@/issue141.tif -o @TEMP_PATH@/issue141-I.rawl.j2k -I
# issue 46: # issue 46:
opj_compress -i @INPUT_NR_PATH@/Bretagne2.ppm -o @TEMP_PATH@/Bretagne2_5.j2k -c [64,64] opj_compress -i @INPUT_NR_PATH@/Bretagne2.ppm -o @TEMP_PATH@/Bretagne2_5.j2k -c [64,64]
# issue 316 # issue 316

View File

@ -1,8 +0,0 @@
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
NR-ENC-issue141.rawl-23-compare_dec-ref-out2base

View File

@ -1,8 +0,0 @@
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
NR-ENC-issue141.rawl-23-compare_dec-ref-out2base

View File

@ -46,11 +46,3 @@ NR-DEC-kodak_2layers_lrcp.j2c-32-decode-md5
NR-DEC-issue135.j2k-68-decode-md5 NR-DEC-issue135.j2k-68-decode-md5
NR-DEC-db11217111510058.jp2-306-decode-md5 NR-DEC-db11217111510058.jp2-306-decode-md5
NR-DEC-tnsot_zero.jp2-307-decode-md5 NR-DEC-tnsot_zero.jp2-307-decode-md5
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
NR-ENC-issue141.rawl-23-compare_dec-ref-out2base

View File

@ -1,8 +0,0 @@
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-15-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-16-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-17-compare_dec-ref-out2base
NR-ENC-X_4_2K_24_185_CBR_WB_000.tif-18-compare_dec-ref-out2base
NR-ENC-X_5_2K_24_235_CBR_STEM24_000.tif-19-compare_dec-ref-out2base
NR-ENC-X_6_2K_24_FULL_CBR_CIRCLE_000.tif-20-compare_dec-ref-out2base
NR-ENC-ElephantDream_4K.tif-21-compare_dec-ref-out2base
NR-ENC-issue141.rawl-23-compare_dec-ref-out2base