Merge pull request #832 from mayeut/overflow-check

Add overflow checks for opj_aligned_malloc
This commit is contained in:
Antonin Descampe 2016-09-15 11:12:58 +02:00 committed by GitHub
commit d8ae42d808
3 changed files with 82 additions and 17 deletions

View File

@ -407,11 +407,19 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
l_cur_res = tilec->resolutions + l; l_cur_res = tilec->resolutions + l;
l_last_res = l_cur_res - 1; l_last_res = l_cur_res - 1;
l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32); l_data_size = opj_dwt_max_resolution(tilec->resolutions, tilec->numresolutions);
bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size);
/* overflow check */
if ((size_t)l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size * sizeof(OPJ_INT32));
/* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */ /* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
/* in that case, so do not error out */ /* in that case, so do not error out */
if (l_data_size != 0 && ! bj) { if (l_data_size != 0 && ! bj) {
/* FIXME event manager error callback */
return OPJ_FALSE; return OPJ_FALSE;
} }
i = l; i = l;
@ -569,11 +577,22 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres
OPJ_UINT32 rh = (OPJ_UINT32)(tr->y1 - tr->y0); /* height of the resolution level computed */ OPJ_UINT32 rh = (OPJ_UINT32)(tr->y1 - tr->y0); /* height of the resolution level computed */
OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
OPJ_UINT32 mr; /* max resolution */
if (numres == 1U) { if (numres == 1U) {
return OPJ_TRUE; return OPJ_TRUE;
} }
h.mem = (OPJ_INT32*)opj_aligned_malloc(opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32));
mr = opj_dwt_max_resolution(tr, numres);
/* overflow check */
if ((size_t)mr > (SIZE_MAX / sizeof(OPJ_INT32))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
h.mem = (OPJ_INT32*)opj_aligned_malloc((size_t)mr * sizeof(OPJ_INT32));
if (! h.mem){ if (! h.mem){
/* FIXME event manager error callback */ /* FIXME event manager error callback */
return OPJ_FALSE; return OPJ_FALSE;
@ -845,8 +864,24 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
OPJ_UINT32 rh = (OPJ_UINT32)(res->y1 - res->y0); /* height of the resolution level computed */ OPJ_UINT32 rh = (OPJ_UINT32)(res->y1 - res->y0); /* height of the resolution level computed */
OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
OPJ_UINT32 mr; /* max resolution */
mr = opj_dwt_max_resolution(res, numres);
/* overflow check */
if (mr > (0xFFFFFFFFU /* UINT32_MAX */ - 5U)) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
mr += 5U;
if ((size_t)mr > (SIZE_MAX / sizeof(opj_v4_t))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
h.wavelet = (opj_v4_t*) opj_aligned_malloc((opj_dwt_max_resolution(res, numres)+5) * sizeof(opj_v4_t)); h.wavelet = (opj_v4_t*) opj_aligned_malloc((size_t)mr * sizeof(opj_v4_t));
if (!h.wavelet) { if (!h.wavelet) {
/* FIXME event manager error callback */ /* FIXME event manager error callback */
return OPJ_FALSE; return OPJ_FALSE;

View File

@ -1238,14 +1238,14 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image,
/* memory allocation for include */ /* memory allocation for include */
/* prevent an integer overflow issue */ /* prevent an integer overflow issue */
/* 0 < l_tcp->numlayers < 65536 c.f. opj_j2k_read_cod in j2k.c */
l_current_pi->include = 00; l_current_pi->include = 00;
if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U))) if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U)))
{ {
l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16)); l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16));
} }
if if (!l_current_pi->include)
(!l_current_pi->include)
{ {
opj_free(l_tmp_data); opj_free(l_tmp_data);
opj_free(l_tmp_ptr); opj_free(l_tmp_ptr);

View File

@ -1166,41 +1166,71 @@ static OPJ_BOOL opj_t1_allocate_buffers(
OPJ_UINT32 w, OPJ_UINT32 w,
OPJ_UINT32 h) OPJ_UINT32 h)
{ {
OPJ_UINT32 datasize=w * h;
OPJ_UINT32 flagssize; OPJ_UINT32 flagssize;
/* encoder uses tile buffer, so no need to allocate */ /* encoder uses tile buffer, so no need to allocate */
if (!t1->encoder) { if (!t1->encoder) {
OPJ_UINT32 datasize;
/* Overflow check */
if ((w > 0U) && (h > (0xFFFFFFFFU /* UINT32_MAX */ / w))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
datasize = w * h;
/* Overflow check */
if ((size_t)datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
if(datasize > t1->datasize){ if(datasize > t1->datasize){
opj_aligned_free(t1->data); opj_aligned_free(t1->data);
t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32)); t1->data = (OPJ_INT32*) opj_aligned_malloc((size_t)datasize * sizeof(OPJ_INT32));
if(!t1->data){ if(!t1->data){
/* FIXME event manager error callback */ /* FIXME event manager error callback */
return OPJ_FALSE; return OPJ_FALSE;
} }
t1->datasize=datasize; t1->datasize = datasize;
} }
/* memset first arg is declared to never be null by gcc */ /* memset first arg is declared to never be null by gcc */
if (t1->data != NULL) { if (t1->data != NULL) {
memset(t1->data,0,datasize * sizeof(OPJ_INT32)); memset(t1->data, 0, (size_t)datasize * sizeof(OPJ_INT32));
} }
} }
t1->flags_stride=w+2;
flagssize=t1->flags_stride * (h+2); if ((w > (0xFFFFFFFFU /* UINT32_MAX */ - 2U)) || (h > (0xFFFFFFFFU /* UINT32_MAX */ - 2U))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
t1->flags_stride = w + 2U; /* can't be 0U */
if ((h + 2U) > (0xFFFFFFFFU /* UINT32_MAX */ / t1->flags_stride)) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
flagssize = t1->flags_stride * (h + 2U);
/* Overflow check */
if ((size_t)flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
/* FIXME event manager error callback */
return OPJ_FALSE;
}
if(flagssize > t1->flagssize){ if(flagssize > t1->flagssize){
opj_aligned_free(t1->flags); opj_aligned_free(t1->flags);
t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t)); t1->flags = (opj_flag_t*) opj_aligned_malloc((size_t)flagssize * sizeof(opj_flag_t));
if(!t1->flags){ if(!t1->flags){
/* FIXME event manager error callback */ /* FIXME event manager error callback */
return OPJ_FALSE; return OPJ_FALSE;
} }
t1->flagssize=flagssize; t1->flagssize = flagssize;
} }
memset(t1->flags,0,flagssize * sizeof(opj_flag_t)); memset(t1->flags, 0, (size_t)flagssize * sizeof(opj_flag_t));
t1->w=w; t1->w = w;
t1->h=h; t1->h = h;
return OPJ_TRUE; return OPJ_TRUE;
} }