Add overflow checks for opj_aligned_malloc (#841)

See
https://pdfium.googlesource.com/pdfium/+/b20ab6c7acb3be1393461eb650ca8fa4660c937e/third_party/libopenjpeg20/0020-opj_aligned_malloc.patch

src/lib/openjp2/dwt.c

@@ -395,7 +395,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 
 	OPJ_INT32 rw;			/* width of the resolution level computed   */
 	OPJ_INT32 rh;			/* height of the resolution level computed  */
-	OPJ_UINT32 l_data_size;
+	size_t l_data_size;
 
 	opj_tcd_resolution_t * l_cur_res = 0;
 	opj_tcd_resolution_t * l_last_res = 0;
@@ -407,8 +407,14 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 	l_cur_res = tilec->resolutions + l;
 	l_last_res = l_cur_res - 1;
 
-	l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32);
+	l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions);
-	bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size);
+	/* overflow check */
+	if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+		/* FIXME event manager error callback */
+		return OPJ_FALSE;
+	}
+	l_data_size *= sizeof(OPJ_INT32);
+	bj = (OPJ_INT32*)opj_malloc(l_data_size);
 	/* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
 	/* in that case, so do not error out */
 	if (l_data_size != 0 && ! bj) {
@@ -638,7 +644,13 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* t
 		return OPJ_TRUE;
 	}
 	num_threads = opj_thread_pool_get_thread_count(tp);
-	h_mem_size = opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32);
+	h_mem_size = opj_dwt_max_resolution(tr, numres);
+	/* overflow check */
+	if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+		/* FIXME event manager error callback */
+		return OPJ_FALSE;
+	}
+	h_mem_size *= sizeof(OPJ_INT32);
 	h.mem = (OPJ_INT32*)opj_aligned_malloc(h_mem_size);
 	if (! h.mem){
 		/* FIXME event manager error callback */
@@ -1003,7 +1015,21 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
 
 	OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
 
-	h.wavelet = (opj_v4_t*) opj_aligned_malloc((opj_dwt_max_resolution(res, numres)+5) * sizeof(opj_v4_t));
+	size_t l_data_size;
+	
+	l_data_size = opj_dwt_max_resolution(res, numres);
+	/* overflow check */
+	if (l_data_size > (SIZE_MAX - 5U)) {
+		/* FIXME event manager error callback */
+		return OPJ_FALSE;
+	}
+	l_data_size += 5U;
+	/* overflow check */
+	if (l_data_size > (SIZE_MAX / sizeof(opj_v4_t))) {
+		/* FIXME event manager error callback */
+		return OPJ_FALSE;
+	}
+	h.wavelet = (opj_v4_t*) opj_aligned_malloc(l_data_size * sizeof(opj_v4_t));
 	if (!h.wavelet) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;

src/lib/openjp2/pi.c

@@ -1238,14 +1238,14 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image,
 
 	/* memory allocation for include */
 	/* prevent an integer overflow issue */
+	/* 0 < l_tcp->numlayers < 65536 c.f. opj_j2k_read_cod in j2k.c */
 	l_current_pi->include = 00;
 	if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U)))
 	{
 		l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16));
 	}
 
-	if
+	if (!l_current_pi->include)
-		(!l_current_pi->include)
 	{
 		opj_free(l_tmp_data);
 		opj_free(l_tmp_ptr);

src/lib/openjp2/t1.c

@@ -1406,56 +1406,135 @@ static OPJ_BOOL opj_t1_allocate_buffers(
 		OPJ_UINT32 w,
 		OPJ_UINT32 h)
 {
-	OPJ_UINT32 datasize=w * h;
-	OPJ_UINT32 flagssize;
-
 	/* encoder uses tile buffer, so no need to allocate */
 	if (!t1->encoder) {
-		if(datasize > t1->datasize){
+		size_t datasize;
+		
+#if (SIZE_MAX / 0xFFFFFFFFU) < 0xFFFFFFFFU /* UINT32_MAX */
+		/* Overflow check */
+		if ((w > 0U) && ((size_t)h > (SIZE_MAX / (size_t)w))) {
+			/* FIXME event manager error callback */
+			return OPJ_FALSE;
+		}
+#endif
+		datasize = (size_t)w * h;
+
+		/* Overflow check */
+		if (datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
+			/* FIXME event manager error callback */
+			return OPJ_FALSE;
+		}
+
+		if(datasize > (size_t)t1->datasize){
 			opj_aligned_free(t1->data);
 			t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32));
 			if(!t1->data){
 				/* FIXME event manager error callback */
 				return OPJ_FALSE;
 			}
-			t1->datasize=datasize;
+#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */
+			/* TODO remove this if t1->datasize type changes to size_t */
+			/* Overflow check */
+			if (datasize > (size_t)0xFFFFFFFFU /* UINT32_MAX */) {
+				/* FIXME event manager error callback */
+				return OPJ_FALSE;
+			}
+#endif
+			t1->datasize = (OPJ_UINT32)datasize;
 		}
 		/* memset first arg is declared to never be null by gcc */
 		if (t1->data != NULL) {
-			memset(t1->data,0,datasize * sizeof(OPJ_INT32));
+			memset(t1->data, 0, datasize * sizeof(OPJ_INT32));
 		}
 	}
-	t1->flags_stride=w+2;
-	flagssize=t1->flags_stride * (h+2);
 
-	if(flagssize > t1->flagssize){
+	{
-		opj_aligned_free(t1->flags);
+		size_t flagssize;
-		t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t));
+
-		if(!t1->flags){
+		/* Overflow check */
+		if (w > (0xFFFFFFFFU /* UINT32_MAX */ - 2U)) {
 			/* FIXME event manager error callback */
 			return OPJ_FALSE;
 		}
-		t1->flagssize=flagssize;
+		t1->flags_stride = w + 2U; /* can't be 0U */
-	}
-	memset(t1->flags,0,flagssize * sizeof(opj_flag_t));
-	
-	if (!t1->encoder) {
-		OPJ_UINT32 colflags_size=t1->flags_stride * ((h+3) / 4 + 2);
 
-		if(colflags_size > t1->colflags_size){
+#if (SIZE_MAX - 3U) < 0xFFFFFFFFU /* UINT32_MAX */
+		/* Overflow check */
+		if (h > (0xFFFFFFFFU /* UINT32_MAX */ - 3U)) {
+			/* FIXME event manager error callback */
+			return OPJ_FALSE;
+		}
+#endif
+		flagssize = (size_t)h + 3U;
+		
+		/* Overflow check */
+		if (flagssize > (SIZE_MAX / (size_t)t1->flags_stride)) {
+			/* FIXME event manager error callback */
+			return OPJ_FALSE;
+		}
+		flagssize *= (size_t)t1->flags_stride;
+
+		if(flagssize > (size_t)t1->flagssize){
+			/* Overflow check */
+			if (flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
+				/* FIXME event manager error callback */
+				return OPJ_FALSE;
+			}
+			opj_aligned_free(t1->flags);
+			t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t));
+			if(!t1->flags){
+				/* FIXME event manager error callback */
+				return OPJ_FALSE;
+			}
+#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */
+			/* TODO remove this if t1->flagssize type changes to size_t */
+			/* Overflow check */
+			if (flagssize > (size_t)0xFFFFFFFFU /* UINT32_MAX */) {
+				/* FIXME event manager error callback */
+				return OPJ_FALSE;
+			}
+#endif
+			t1->flagssize = (OPJ_UINT32)flagssize;
+		}
+		memset(t1->flags, 0, flagssize * sizeof(opj_flag_t));
+	}
+	if (!t1->encoder) {
+		size_t colflags_size = ((((size_t)h + 3U) / 4U) + 2U); /* Can't overflow, h checked against UINT32_MAX - 3U */
+		
+		/* Overflow check */
+		if (colflags_size > (SIZE_MAX / (size_t)t1->flags_stride)) {
+			/* FIXME event manager error callback */
+			return OPJ_FALSE;
+		}
+		colflags_size *= (size_t)t1->flags_stride;
+		
+		if(colflags_size > (size_t)t1->colflags_size){
+			/* Overflow check */
+			if ((size_t)colflags_size > (SIZE_MAX / sizeof(opj_colflag_t))) {
+				/* FIXME event manager error callback */
+				return OPJ_FALSE;
+			}
 			opj_aligned_free(t1->colflags);
 			t1->colflags = (opj_colflag_t*) opj_aligned_malloc(colflags_size * sizeof(opj_colflag_t));
 			if(!t1->colflags){
 				/* FIXME event manager error callback */
 				return OPJ_FALSE;
 			}
-			t1->colflags_size=colflags_size;
+#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */
+			/* TODO remove this if t1->colflags_size type changes to size_t */
+			/* Overflow check */
+			if (colflags_size > (size_t)0xFFFFFFFFU /* UINT32_MAX */) {
+				/* FIXME event manager error callback */
+				return OPJ_FALSE;
+			}
+#endif
+			t1->colflags_size = (OPJ_UINT32)colflags_size;
 		}
-		memset(t1->colflags,0,colflags_size * sizeof(opj_colflag_t));
+		memset(t1->colflags, 0, colflags_size * sizeof(opj_colflag_t));
 	}
 
-	t1->w=w;
+	t1->w = w;
-	t1->h=h;
+	t1->h = h;
 
 	return OPJ_TRUE;
 }