From 43557dcd3bac636283f5205089fe767eae207fb0 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 12 Sep 2016 20:20:57 +0200 Subject: [PATCH] Add overflow checks for opj_aligned_malloc See https://pdfium.googlesource.com/pdfium/+/b20ab6c7acb3be1393461eb650ca8fa 4660c937e/third_party/libopenjpeg20/0020-opj_aligned_malloc.patch --- src/lib/openjp2/dwt.c | 43 +++++++++++++++++++++++++++++++---- src/lib/openjp2/pi.c | 4 ++-- src/lib/openjp2/t1.c | 52 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 82 insertions(+), 17 deletions(-) diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c index 4fce8b20..b39f63e7 100644 --- a/src/lib/openjp2/dwt.c +++ b/src/lib/openjp2/dwt.c @@ -407,11 +407,19 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void l_cur_res = tilec->resolutions + l; l_last_res = l_cur_res - 1; - l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32); - bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size); + l_data_size = opj_dwt_max_resolution(tilec->resolutions, tilec->numresolutions); + + /* overflow check */ + if ((size_t)l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + + bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size * sizeof(OPJ_INT32)); /* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */ /* in that case, so do not error out */ if (l_data_size != 0 && ! bj) { + /* FIXME event manager error callback */ return OPJ_FALSE; } i = l; @@ -569,11 +577,22 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres OPJ_UINT32 rh = (OPJ_UINT32)(tr->y1 - tr->y0); /* height of the resolution level computed */ OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); + + OPJ_UINT32 mr; /* max resolution */ if (numres == 1U) { return OPJ_TRUE; } - h.mem = (OPJ_INT32*)opj_aligned_malloc(opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32)); + + mr = opj_dwt_max_resolution(tr, numres); + + /* overflow check */ + if ((size_t)mr > (SIZE_MAX / sizeof(OPJ_INT32))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + + h.mem = (OPJ_INT32*)opj_aligned_malloc((size_t)mr * sizeof(OPJ_INT32)); if (! h.mem){ /* FIXME event manager error callback */ return OPJ_FALSE; @@ -845,8 +864,24 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32 OPJ_UINT32 rh = (OPJ_UINT32)(res->y1 - res->y0); /* height of the resolution level computed */ OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); + + OPJ_UINT32 mr; /* max resolution */ + + mr = opj_dwt_max_resolution(res, numres); + + /* overflow check */ + if (mr > (0xFFFFFFFFU /* UINT32_MAX */ - 5U)) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + mr += 5U; + + if ((size_t)mr > (SIZE_MAX / sizeof(opj_v4_t))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } - h.wavelet = (opj_v4_t*) opj_aligned_malloc((opj_dwt_max_resolution(res, numres)+5) * sizeof(opj_v4_t)); + h.wavelet = (opj_v4_t*) opj_aligned_malloc((size_t)mr * sizeof(opj_v4_t)); if (!h.wavelet) { /* FIXME event manager error callback */ return OPJ_FALSE; diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c index 809b33d7..41a2f046 100644 --- a/src/lib/openjp2/pi.c +++ b/src/lib/openjp2/pi.c @@ -1238,14 +1238,14 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image, /* memory allocation for include */ /* prevent an integer overflow issue */ + /* 0 < l_tcp->numlayers < 65536 c.f. opj_j2k_read_cod in j2k.c */ l_current_pi->include = 00; if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U))) { l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16)); } - if - (!l_current_pi->include) + if (!l_current_pi->include) { opj_free(l_tmp_data); opj_free(l_tmp_ptr); diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c index cb5a1cef..211708fc 100644 --- a/src/lib/openjp2/t1.c +++ b/src/lib/openjp2/t1.c @@ -1166,41 +1166,71 @@ static OPJ_BOOL opj_t1_allocate_buffers( OPJ_UINT32 w, OPJ_UINT32 h) { - OPJ_UINT32 datasize=w * h; OPJ_UINT32 flagssize; /* encoder uses tile buffer, so no need to allocate */ if (!t1->encoder) { + OPJ_UINT32 datasize; + + /* Overflow check */ + if ((w > 0U) && (h > (0xFFFFFFFFU /* UINT32_MAX */ / w))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + datasize = w * h; + + /* Overflow check */ + if ((size_t)datasize > (SIZE_MAX / sizeof(OPJ_INT32))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + if(datasize > t1->datasize){ opj_aligned_free(t1->data); - t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32)); + t1->data = (OPJ_INT32*) opj_aligned_malloc((size_t)datasize * sizeof(OPJ_INT32)); if(!t1->data){ /* FIXME event manager error callback */ return OPJ_FALSE; } - t1->datasize=datasize; + t1->datasize = datasize; } /* memset first arg is declared to never be null by gcc */ if (t1->data != NULL) { - memset(t1->data,0,datasize * sizeof(OPJ_INT32)); + memset(t1->data, 0, (size_t)datasize * sizeof(OPJ_INT32)); } } - t1->flags_stride=w+2; - flagssize=t1->flags_stride * (h+2); + + if ((w > (0xFFFFFFFFU /* UINT32_MAX */ - 2U)) || (h > (0xFFFFFFFFU /* UINT32_MAX */ - 2U))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + + t1->flags_stride = w + 2U; /* can't be 0U */ + if ((h + 2U) > (0xFFFFFFFFU /* UINT32_MAX */ / t1->flags_stride)) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + flagssize = t1->flags_stride * (h + 2U); + /* Overflow check */ + if ((size_t)flagssize > (SIZE_MAX / sizeof(opj_flag_t))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + if(flagssize > t1->flagssize){ opj_aligned_free(t1->flags); - t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t)); + t1->flags = (opj_flag_t*) opj_aligned_malloc((size_t)flagssize * sizeof(opj_flag_t)); if(!t1->flags){ /* FIXME event manager error callback */ return OPJ_FALSE; } - t1->flagssize=flagssize; + t1->flagssize = flagssize; } - memset(t1->flags,0,flagssize * sizeof(opj_flag_t)); + memset(t1->flags, 0, (size_t)flagssize * sizeof(opj_flag_t)); - t1->w=w; - t1->h=h; + t1->w = w; + t1->h = h; return OPJ_TRUE; }