From 25966346f76e1e27adf35a7180cddf311a3c9788 Mon Sep 17 00:00:00 2001 From: Antonin Descampe Date: Fri, 16 Sep 2016 17:48:20 +0200 Subject: [PATCH] Fix warnings from recent overflow checks see #832 --- src/lib/openjp2/dwt.c | 23 +++++++++++++---------- src/lib/openjp2/t1.c | 28 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c index b39f63e7..b7d5dc2f 100644 --- a/src/lib/openjp2/dwt.c +++ b/src/lib/openjp2/dwt.c @@ -395,7 +395,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void OPJ_INT32 rw; /* width of the resolution level computed */ OPJ_INT32 rh; /* height of the resolution level computed */ - OPJ_UINT32 l_data_size; + size_t l_data_size; opj_tcd_resolution_t * l_cur_res = 0; opj_tcd_resolution_t * l_last_res = 0; @@ -410,12 +410,13 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void l_data_size = opj_dwt_max_resolution(tilec->resolutions, tilec->numresolutions); /* overflow check */ - if ((size_t)l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) { + if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) { /* FIXME event manager error callback */ return OPJ_FALSE; } - bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size * sizeof(OPJ_INT32)); + l_data_size *= sizeof(OPJ_INT32); + bj = (OPJ_INT32*)opj_malloc(l_data_size); /* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */ /* in that case, so do not error out */ if (l_data_size != 0 && ! bj) { @@ -578,21 +579,22 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); - OPJ_UINT32 mr; /* max resolution */ + size_t mr; /* max resolution */ if (numres == 1U) { return OPJ_TRUE; } - mr = opj_dwt_max_resolution(tr, numres); + mr = opj_dwt_max_resolution(tr, numres); /* overflow check */ - if ((size_t)mr > (SIZE_MAX / sizeof(OPJ_INT32))) { + if (mr > (SIZE_MAX / sizeof(OPJ_INT32))) { /* FIXME event manager error callback */ return OPJ_FALSE; } - h.mem = (OPJ_INT32*)opj_aligned_malloc((size_t)mr * sizeof(OPJ_INT32)); + mr *= sizeof(OPJ_INT32); + h.mem = (OPJ_INT32*)opj_aligned_malloc(mr); if (! h.mem){ /* FIXME event manager error callback */ return OPJ_FALSE; @@ -865,7 +867,7 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32 OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0); - OPJ_UINT32 mr; /* max resolution */ + size_t mr; /* max resolution */ mr = opj_dwt_max_resolution(res, numres); @@ -876,12 +878,13 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32 } mr += 5U; - if ((size_t)mr > (SIZE_MAX / sizeof(opj_v4_t))) { + if (mr > (SIZE_MAX / sizeof(opj_v4_t))) { /* FIXME event manager error callback */ return OPJ_FALSE; } - h.wavelet = (opj_v4_t*) opj_aligned_malloc((size_t)mr * sizeof(opj_v4_t)); + mr *= sizeof(opj_v4_t); + h.wavelet = (opj_v4_t*) opj_aligned_malloc(mr); if (!h.wavelet) { /* FIXME event manager error callback */ return OPJ_FALSE; diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c index 211708fc..bd3acf7e 100644 --- a/src/lib/openjp2/t1.c +++ b/src/lib/openjp2/t1.c @@ -1166,37 +1166,37 @@ static OPJ_BOOL opj_t1_allocate_buffers( OPJ_UINT32 w, OPJ_UINT32 h) { - OPJ_UINT32 flagssize; + size_t flagssize; /* encoder uses tile buffer, so no need to allocate */ if (!t1->encoder) { - OPJ_UINT32 datasize; + size_t datasize; /* Overflow check */ if ((w > 0U) && (h > (0xFFFFFFFFU /* UINT32_MAX */ / w))) { /* FIXME event manager error callback */ return OPJ_FALSE; } - datasize = w * h; + datasize = (size_t) w * h; /* Overflow check */ - if ((size_t)datasize > (SIZE_MAX / sizeof(OPJ_INT32))) { + if (datasize > (SIZE_MAX / sizeof(OPJ_INT32))) { /* FIXME event manager error callback */ return OPJ_FALSE; } - if(datasize > t1->datasize){ + if(datasize > (size_t) t1->datasize){ opj_aligned_free(t1->data); - t1->data = (OPJ_INT32*) opj_aligned_malloc((size_t)datasize * sizeof(OPJ_INT32)); + t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32)); if(!t1->data){ /* FIXME event manager error callback */ return OPJ_FALSE; } - t1->datasize = datasize; + t1->datasize = (OPJ_UINT32) datasize; } /* memset first arg is declared to never be null by gcc */ if (t1->data != NULL) { - memset(t1->data, 0, (size_t)datasize * sizeof(OPJ_INT32)); + memset(t1->data, 0, datasize * sizeof(OPJ_INT32)); } } @@ -1210,24 +1210,24 @@ static OPJ_BOOL opj_t1_allocate_buffers( /* FIXME event manager error callback */ return OPJ_FALSE; } - flagssize = t1->flags_stride * (h + 2U); + flagssize = (size_t) t1->flags_stride * (h + 2U); /* Overflow check */ - if ((size_t)flagssize > (SIZE_MAX / sizeof(opj_flag_t))) { + if (flagssize > (SIZE_MAX / sizeof(opj_flag_t))) { /* FIXME event manager error callback */ return OPJ_FALSE; } - if(flagssize > t1->flagssize){ + if(flagssize > (size_t) t1->flagssize){ opj_aligned_free(t1->flags); - t1->flags = (opj_flag_t*) opj_aligned_malloc((size_t)flagssize * sizeof(opj_flag_t)); + t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t)); if(!t1->flags){ /* FIXME event manager error callback */ return OPJ_FALSE; } - t1->flagssize = flagssize; + t1->flagssize = (OPJ_UINT32) flagssize; } - memset(t1->flags, 0, (size_t)flagssize * sizeof(opj_flag_t)); + memset(t1->flags, 0, flagssize * sizeof(opj_flag_t)); t1->w = w; t1->h = h;