From 9a07ccb3d0f076388e4da684a3bfd4327125c721 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Thu, 15 Sep 2016 01:57:53 +0200 Subject: [PATCH] Add overflow checks for opj_aligned_malloc (#841) See https://pdfium.googlesource.com/pdfium/+/b20ab6c7acb3be1393461eb650ca8fa4660c937e/third_party/libopenjpeg20/0020-opj_aligned_malloc.patch --- src/lib/openjp2/dwt.c | 36 ++++++++++-- src/lib/openjp2/pi.c | 4 +- src/lib/openjp2/t1.c | 125 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 135 insertions(+), 30 deletions(-) diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c index 2d793bb6..98ead59e 100644 --- a/src/lib/openjp2/dwt.c +++ b/src/lib/openjp2/dwt.c @@ -395,7 +395,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void OPJ_INT32 rw; /* width of the resolution level computed */ OPJ_INT32 rh; /* height of the resolution level computed */ - OPJ_UINT32 l_data_size; + size_t l_data_size; opj_tcd_resolution_t * l_cur_res = 0; opj_tcd_resolution_t * l_last_res = 0; @@ -407,8 +407,14 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void l_cur_res = tilec->resolutions + l; l_last_res = l_cur_res - 1; - l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32); - bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size); + l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions); + /* overflow check */ + if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + l_data_size *= sizeof(OPJ_INT32); + bj = (OPJ_INT32*)opj_malloc(l_data_size); /* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */ /* in that case, so do not error out */ if (l_data_size != 0 && ! bj) { @@ -638,7 +644,13 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* t return OPJ_TRUE; } num_threads = opj_thread_pool_get_thread_count(tp); - h_mem_size = opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32); + h_mem_size = opj_dwt_max_resolution(tr, numres); + /* overflow check */ + if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + h_mem_size *= sizeof(OPJ_INT32); h.mem = (OPJ_INT32*)opj_aligned_malloc(h_mem_size); if (! h.mem){ /* FIXME event manager error callback */ @@ -1003,7 +1015,21 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32 OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); - h.wavelet = (opj_v4_t*) opj_aligned_malloc((opj_dwt_max_resolution(res, numres)+5) * sizeof(opj_v4_t)); + size_t l_data_size; + + l_data_size = opj_dwt_max_resolution(res, numres); + /* overflow check */ + if (l_data_size > (SIZE_MAX - 5U)) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + l_data_size += 5U; + /* overflow check */ + if (l_data_size > (SIZE_MAX / sizeof(opj_v4_t))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + h.wavelet = (opj_v4_t*) opj_aligned_malloc(l_data_size * sizeof(opj_v4_t)); if (!h.wavelet) { /* FIXME event manager error callback */ return OPJ_FALSE; diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c index 809b33d7..41a2f046 100644 --- a/src/lib/openjp2/pi.c +++ b/src/lib/openjp2/pi.c @@ -1238,14 +1238,14 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image, /* memory allocation for include */ /* prevent an integer overflow issue */ + /* 0 < l_tcp->numlayers < 65536 c.f. opj_j2k_read_cod in j2k.c */ l_current_pi->include = 00; if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U))) { l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16)); } - if - (!l_current_pi->include) + if (!l_current_pi->include) { opj_free(l_tmp_data); opj_free(l_tmp_ptr); diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c index 53451677..453d2908 100644 --- a/src/lib/openjp2/t1.c +++ b/src/lib/openjp2/t1.c @@ -1406,56 +1406,135 @@ static OPJ_BOOL opj_t1_allocate_buffers( OPJ_UINT32 w, OPJ_UINT32 h) { - OPJ_UINT32 datasize=w * h; - OPJ_UINT32 flagssize; - /* encoder uses tile buffer, so no need to allocate */ if (!t1->encoder) { - if(datasize > t1->datasize){ + size_t datasize; + +#if (SIZE_MAX / 0xFFFFFFFFU) < 0xFFFFFFFFU /* UINT32_MAX */ + /* Overflow check */ + if ((w > 0U) && ((size_t)h > (SIZE_MAX / (size_t)w))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } +#endif + datasize = (size_t)w * h; + + /* Overflow check */ + if (datasize > (SIZE_MAX / sizeof(OPJ_INT32))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + + if(datasize > (size_t)t1->datasize){ opj_aligned_free(t1->data); t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32)); if(!t1->data){ /* FIXME event manager error callback */ return OPJ_FALSE; } - t1->datasize=datasize; +#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */ + /* TODO remove this if t1->datasize type changes to size_t */ + /* Overflow check */ + if (datasize > (size_t)0xFFFFFFFFU /* UINT32_MAX */) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } +#endif + t1->datasize = (OPJ_UINT32)datasize; } /* memset first arg is declared to never be null by gcc */ if (t1->data != NULL) { - memset(t1->data,0,datasize * sizeof(OPJ_INT32)); + memset(t1->data, 0, datasize * sizeof(OPJ_INT32)); } } - t1->flags_stride=w+2; - flagssize=t1->flags_stride * (h+2); - if(flagssize > t1->flagssize){ - opj_aligned_free(t1->flags); - t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t)); - if(!t1->flags){ + { + size_t flagssize; + + /* Overflow check */ + if (w > (0xFFFFFFFFU /* UINT32_MAX */ - 2U)) { /* FIXME event manager error callback */ return OPJ_FALSE; } - t1->flagssize=flagssize; - } - memset(t1->flags,0,flagssize * sizeof(opj_flag_t)); - - if (!t1->encoder) { - OPJ_UINT32 colflags_size=t1->flags_stride * ((h+3) / 4 + 2); + t1->flags_stride = w + 2U; /* can't be 0U */ - if(colflags_size > t1->colflags_size){ +#if (SIZE_MAX - 3U) < 0xFFFFFFFFU /* UINT32_MAX */ + /* Overflow check */ + if (h > (0xFFFFFFFFU /* UINT32_MAX */ - 3U)) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } +#endif + flagssize = (size_t)h + 3U; + + /* Overflow check */ + if (flagssize > (SIZE_MAX / (size_t)t1->flags_stride)) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + flagssize *= (size_t)t1->flags_stride; + + if(flagssize > (size_t)t1->flagssize){ + /* Overflow check */ + if (flagssize > (SIZE_MAX / sizeof(opj_flag_t))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + opj_aligned_free(t1->flags); + t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t)); + if(!t1->flags){ + /* FIXME event manager error callback */ + return OPJ_FALSE; + } +#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */ + /* TODO remove this if t1->flagssize type changes to size_t */ + /* Overflow check */ + if (flagssize > (size_t)0xFFFFFFFFU /* UINT32_MAX */) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } +#endif + t1->flagssize = (OPJ_UINT32)flagssize; + } + memset(t1->flags, 0, flagssize * sizeof(opj_flag_t)); + } + if (!t1->encoder) { + size_t colflags_size = ((((size_t)h + 3U) / 4U) + 2U); /* Can't overflow, h checked against UINT32_MAX - 3U */ + + /* Overflow check */ + if (colflags_size > (SIZE_MAX / (size_t)t1->flags_stride)) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } + colflags_size *= (size_t)t1->flags_stride; + + if(colflags_size > (size_t)t1->colflags_size){ + /* Overflow check */ + if ((size_t)colflags_size > (SIZE_MAX / sizeof(opj_colflag_t))) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } opj_aligned_free(t1->colflags); t1->colflags = (opj_colflag_t*) opj_aligned_malloc(colflags_size * sizeof(opj_colflag_t)); if(!t1->colflags){ /* FIXME event manager error callback */ return OPJ_FALSE; } - t1->colflags_size=colflags_size; +#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */ + /* TODO remove this if t1->colflags_size type changes to size_t */ + /* Overflow check */ + if (colflags_size > (size_t)0xFFFFFFFFU /* UINT32_MAX */) { + /* FIXME event manager error callback */ + return OPJ_FALSE; + } +#endif + t1->colflags_size = (OPJ_UINT32)colflags_size; } - memset(t1->colflags,0,colflags_size * sizeof(opj_colflag_t)); + memset(t1->colflags, 0, colflags_size * sizeof(opj_colflag_t)); } - t1->w=w; - t1->h=h; + t1->w = w; + t1->h = h; return OPJ_TRUE; }