walkero
/
openjpeg
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Subtile decoding: fix overflows in subband coordinate computation that cause later buffer overflow. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=3115. Credit to OSS Fuzz. master only

This commit is contained in:
Even Rouault 2017-08-28 17:18:33 +02:00
parent 8f92fc9791
commit a55c024fc6
2 changed files with 27 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/lib/openjp2/dwt.c
View File

@ -1642,20 +1642,24 @@ static void opj_dwt_get_band_coordinates(opj_tcd_tilecomp_t* tilec,
OPJ_UINT32 x0b = bandno & 1; OPJ_UINT32 x0b = bandno & 1;
OPJ_UINT32 y0b = bandno >> 1; OPJ_UINT32 y0b = bandno >> 1;
if (tbx0) { if (tbx0) {
*tbx0 = (nb == 0) ? tcx0 : opj_uint_ceildiv(tcx0 - (1U << *tbx0 = (nb == 0) ? tcx0 :
(nb - 1)) * x0b, 1U << nb); (tcx0 <= (1U << (nb - 1)) * x0b) ? 0 :
opj_uint_ceildivpow2(tcx0 - (1U << (nb - 1)) * x0b, nb);
} }
if (tby0) { if (tby0) {
*tby0 = (nb == 0) ? tcy0 : opj_uint_ceildiv(tcy0 - (1U << *tby0 = (nb == 0) ? tcy0 :
(nb - 1)) * y0b, 1U << nb); (tcy0 <= (1U << (nb - 1)) * y0b) ? 0 :
opj_uint_ceildivpow2(tcy0 - (1U << (nb - 1)) * y0b, nb);
} }
if (tbx1) { if (tbx1) {
*tbx1 = (nb == 0) ? tcx1 : opj_uint_ceildiv(tcx1 - (1U << *tbx1 = (nb == 0) ? tcx1 :
(nb - 1)) * x0b, 1U << nb); (tcx1 <= (1U << (nb - 1)) * x0b) ? 0 :
opj_uint_ceildivpow2(tcx1 - (1U << (nb - 1)) * x0b, nb);
} }
if (tby1) { if (tby1) {
*tby1 = (nb == 0) ? tcy1 : opj_uint_ceildiv(tcy1 - (1U << *tby1 = (nb == 0) ? tcy1 :
(nb - 1)) * y0b, 1U << nb); (tcy1 <= (1U << (nb - 1)) * y0b) ? 0 :
opj_uint_ceildivpow2(tcy1 - (1U << (nb - 1)) * y0b, nb);
} }
} }
@ -1711,14 +1715,12 @@ static OPJ_BOOL opj_dwt_decode_partial_tile(opj_tcd_t *tcd,
} }
h_mem_size = opj_dwt_max_resolution(tr, numres); h_mem_size = opj_dwt_max_resolution(tr, numres);
/* overflow check */ /* overflow check */
if (h_mem_size > (SIZE_MAX / PARALLEL_COLS_53 / sizeof(OPJ_INT32))) { if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
/* FIXME event manager error callback */ /* FIXME event manager error callback */
return OPJ_FALSE; return OPJ_FALSE;
} }
/* We need PARALLEL_COLS_53 times the height of the array, */
/* since for the vertical pass */ h_mem_size *= sizeof(OPJ_INT32);
/* we process PARALLEL_COLS_53 columns at a time */
h_mem_size *= PARALLEL_COLS_53 * sizeof(OPJ_INT32);
h.mem = (OPJ_INT32*)opj_aligned_32_malloc(h_mem_size); h.mem = (OPJ_INT32*)opj_aligned_32_malloc(h_mem_size);
if (! h.mem) { if (! h.mem) {
/* FIXME event manager error callback */ /* FIXME event manager error callback */

20
src/lib/openjp2/tcd.c
View File

@ -2414,14 +2414,18 @@ OPJ_BOOL opj_tcd_is_subband_area_of_interest(opj_tcd_t *tcd,
/* equation B-15 of the standard */ /* equation B-15 of the standard */
OPJ_UINT32 x0b = bandno & 1; OPJ_UINT32 x0b = bandno & 1;
OPJ_UINT32 y0b = bandno >> 1; OPJ_UINT32 y0b = bandno >> 1;
OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 : opj_uint_ceildiv(tcx0 - (1U << OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 :
(nb - 1)) * x0b, 1U << nb); (tcx0 <= (1U << (nb - 1)) * x0b) ? 0 :
OPJ_UINT32 tby0 = (nb == 0) ? tcy0 : opj_uint_ceildiv(tcy0 - (1U << opj_uint_ceildivpow2(tcx0 - (1U << (nb - 1)) * x0b, nb);
(nb - 1)) * y0b, 1U << nb); OPJ_UINT32 tby0 = (nb == 0) ? tcy0 :
OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 : opj_uint_ceildiv(tcx1 - (1U << (tcy0 <= (1U << (nb - 1)) * y0b) ? 0 :
(nb - 1)) * x0b, 1U << nb); opj_uint_ceildivpow2(tcy0 - (1U << (nb - 1)) * y0b, nb);
OPJ_UINT32 tby1 = (nb == 0) ? tcy1 : opj_uint_ceildiv(tcy1 - (1U << OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 :
(nb - 1)) * y0b, 1U << nb); (tcx1 <= (1U << (nb - 1)) * x0b) ? 0 :
opj_uint_ceildivpow2(tcx1 - (1U << (nb - 1)) * x0b, nb);
OPJ_UINT32 tby1 = (nb == 0) ? tcy1 :
(tcy1 <= (1U << (nb - 1)) * y0b) ? 0 :
opj_uint_ceildivpow2(tcy1 - (1U << (nb - 1)) * y0b, nb);
OPJ_BOOL intersects; OPJ_BOOL intersects;
if (tbx0 < filter_margin) { if (tbx0 < filter_margin) {