walkero
/
openjpeg
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Fix various undefined shift behaviour in pi.c 

Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2496
Credit to OSS Fuzz
This commit is contained in:
Even Rouault 2017-07-26 22:53:59 +02:00
parent 6c5fe9407b
commit a88cbb6a0b
1 changed files with 54 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
src/lib/openjp2/pi.c
View File

@ -353,12 +353,21 @@ static OPJ_BOOL opj_pi_next_rpcl(opj_pi_iterator_t * pi)
for (resno = 0; resno < comp->numresolutions; resno++) {
OPJ_UINT32 dx, dy;
res = &comp->resolutions[resno];
dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
if (res->pdx + comp->numresolutions - 1 - resno < 32 &&
comp->dx <= UINT_MAX / (1u << (res->pdx + comp->numresolutions - 1 - resno))) {
dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
}
if (res->pdy + comp->numresolutions - 1 - resno < 32 &&
comp->dy <= UINT_MAX / (1u << (res->pdy + comp->numresolutions - 1 - resno))) {
dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
}
}
}
if (pi->dx == 0 || pi->dy == 0) {
return OPJ_FALSE;
}
}
if (!pi->tp_on) {
pi->poc.ty0 = pi->ty0;
@ -386,7 +395,8 @@ static OPJ_BOOL opj_pi_next_rpcl(opj_pi_iterator_t * pi)
/* Avoids division by zero */
/* Relates to id_000004,sig_06,src_000679,op_arith8,pos_49,val_-17 */
/* of https://github.com/uclouvain/openjpeg/issues/938 */
if (((comp->dx << levelno) >> levelno) != comp->dx ||
if (levelno >= 32 ||
((comp->dx << levelno) >> levelno) != comp->dx ||
((comp->dy << levelno) >> levelno) != comp->dy) {
continue;
}
@ -472,12 +482,21 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi)
for (resno = 0; resno < comp->numresolutions; resno++) {
OPJ_UINT32 dx, dy;
res = &comp->resolutions[resno];
dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
if (res->pdx + comp->numresolutions - 1 - resno < 32 &&
comp->dx <= UINT_MAX / (1u << (res->pdx + comp->numresolutions - 1 - resno))) {
dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
}
if (res->pdy + comp->numresolutions - 1 - resno < 32 &&
comp->dy <= UINT_MAX / (1u << (res->pdy + comp->numresolutions - 1 - resno))) {
dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
}
}
}
if (pi->dx == 0 || pi->dy == 0) {
return OPJ_FALSE;
}
}
if (!pi->tp_on) {
pi->poc.ty0 = pi->ty0;
@ -591,10 +610,19 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi)
for (resno = 0; resno < comp->numresolutions; resno++) {
OPJ_UINT32 dx, dy;
res = &comp->resolutions[resno];
dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
if (res->pdx + comp->numresolutions - 1 - resno < 32 &&
comp->dx <= UINT_MAX / (1u << (res->pdx + comp->numresolutions - 1 - resno))) {
dx = comp->dx * (1u << (res->pdx + comp->numresolutions - 1 - resno));
pi->dx = !pi->dx ? dx : opj_uint_min(pi->dx, dx);
}
if (res->pdy + comp->numresolutions - 1 - resno < 32 &&
comp->dy <= UINT_MAX / (1u << (res->pdy + comp->numresolutions - 1 - resno))) {
dy = comp->dy * (1u << (res->pdy + comp->numresolutions - 1 - resno));
pi->dy = !pi->dy ? dy : opj_uint_min(pi->dy, dy);
}
}
if (pi->dx == 0 || pi->dy == 0) {
return OPJ_FALSE;
}
if (!pi->tp_on) {
pi->poc.ty0 = pi->ty0;
@ -617,7 +645,8 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi)
levelno = comp->numresolutions - 1 - pi->resno;
/* Avoids division by zero on id_000004,sig_06,src_000679,op_arith8,pos_49,val_-17 */
/* of https://github.com/uclouvain/openjpeg/issues/938 */
if (((comp->dx << levelno) >> levelno) != comp->dx ||
if (levelno >= 32 ||
((comp->dx << levelno) >> levelno) != comp->dx ||
((comp->dy << levelno) >> levelno) != comp->dy) {
continue;
}
@ -896,11 +925,17 @@ static void opj_get_all_encoding_parameters(const opj_image_t *p_image,
l_pdy = l_tccp->prch[resno];
*lResolutionPtr++ = l_pdx;
*lResolutionPtr++ = l_pdy;
l_dx = l_img_comp->dx * (1u << (l_pdx + l_level_no));
l_dy = l_img_comp->dy * (1u << (l_pdy + l_level_no));
/* take the minimum size for l_dx for each comp and resolution*/
*p_dx_min = (OPJ_UINT32)opj_int_min((OPJ_INT32) * p_dx_min, (OPJ_INT32)l_dx);
*p_dy_min = (OPJ_UINT32)opj_int_min((OPJ_INT32) * p_dy_min, (OPJ_INT32)l_dy);
if (l_pdx + l_level_no < 32 &&
l_img_comp->dx <= UINT_MAX / (1u << (l_pdx + l_level_no))) {
l_dx = l_img_comp->dx * (1u << (l_pdx + l_level_no));
/* take the minimum size for l_dx for each comp and resolution*/
*p_dx_min = (OPJ_UINT32)opj_int_min((OPJ_INT32) * p_dx_min, (OPJ_INT32)l_dx);
}
if (l_pdy + l_level_no < 32 &&
l_img_comp->dy <= UINT_MAX / (1u << (l_pdy + l_level_no))) {
l_dy = l_img_comp->dy * (1u << (l_pdy + l_level_no));
*p_dy_min = (OPJ_UINT32)opj_int_min((OPJ_INT32) * p_dy_min, (OPJ_INT32)l_dy);
}
/* various calculations of extents*/
l_rx0 = opj_int_ceildivpow2(l_tcx0, (OPJ_INT32)l_level_no);