From c196b23b90321b5c7e3238294607a2e8626c503f Mon Sep 17 00:00:00 2001
From: ichlubna <43234438+ichlubna@users.noreply.github.com>
Date: Fri, 16 Nov 2018 09:40:31 +0100
Subject: [PATCH] openjp3d: Int overflow fixed (#1159)

When compressing a lot of slices (starting from 44 FullHD slices with 3 8bit components in our experiments) the rate values are high enough to cause an int overflow that leads to negative lengths and wrong results. The cast happens too late.
---
 src/lib/openjp3d/tcd.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/lib/openjp3d/tcd.c b/src/lib/openjp3d/tcd.c
index 013a9f1a..714a0565 100644
--- a/src/lib/openjp3d/tcd.c
+++ b/src/lib/openjp3d/tcd.c
@@ -226,10 +226,10 @@ void tcd_malloc_encode(opj_tcd_t *tcd, opj_volume_t * volume, opj_cp_t * cp,
         if (tcp->rates[j] <= 1) {
             tcp->rates[j] = 0;
         } else {
-            float num = (float)(tile->numcomps * (tile->x1 - tile->x0) *
-                                (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec);
-            float den = (float)(8 * volume->comps[0].dx * volume->comps[0].dy *
-                                volume->comps[0].dz);
+            float num = (float)tile->numcomps * (tile->x1 - tile->x0) *
+                        (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec;
+            float den = 8.0f * volume->comps[0].dx * volume->comps[0].dy *
+                        volume->comps[0].dz;
             den = tcp->rates[j] * den;
             tcp->rates[j] = (num + den - 1) / den;
         }
@@ -537,10 +537,10 @@ void tcd_init_encode(opj_tcd_t *tcd, opj_volume_t * volume, opj_cp_t * cp,
         if (tcp->rates[j] <= 1) {
             tcp->rates[j] = 0;
         } else {
-            float num = (float)(tile->numcomps * (tile->x1 - tile->x0) *
-                                (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec);
-            float den = (float)(8 * volume->comps[0].dx * volume->comps[0].dy *
-                                volume->comps[0].dz);
+            float num = (float)tile->numcomps * (tile->x1 - tile->x0) *
+                        (tile->y1 - tile->y0) * (tile->z1 - tile->z0) * volume->comps[0].prec;
+            float den = 8.0f * volume->comps[0].dx * volume->comps[0].dy *
+                        volume->comps[0].dz;
             den = tcp->rates[j] * den;
             tcp->rates[j] = (num + den - 1) / den;
         }