Fix incorrect decoding of image with large number of progression levels

Fixes regression introduced per d27ccf01c68a31ad62b33d2dc1ba2bb1eeaafe7b

Fixes #1447

2 files changed, 94 insertions, 88 deletions

diff --git a/src/lib/openjp2/opj_intmath.h b/src/lib/openjp2/opj_intmath.h
index 1b0c9d03..cce7a3ca 100644
--- a/src/lib/openjp2/opj_intmath.h
+++ b/src/lib/openjp2/opj_intmath.h
@@ -174,6 +174,17 @@ static INLINE OPJ_UINT32  opj_uint_ceildiv(OPJ_UINT32  a, OPJ_UINT32  b)
 }
 
 /**
+Divide an integer and round upwards
+@return Returns a divided by b
+*/
+static INLINE OPJ_UINT32  opj_uint64_ceildiv_res_uint32(OPJ_UINT64 a,
+        OPJ_UINT64 b)
+{
+    assert(b);
+    return (OPJ_UINT32)((a + b - 1) / b);
+}
+
+/**
 Divide an integer by a power of 2 and round upwards
 @return Returns a divided by 2^b
 */
diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c
index 38f1ba5a..15ac3314 100644
--- a/src/lib/openjp2/pi.c
+++ b/src/lib/openjp2/pi.c
@@ -411,41 +411,37 @@ static OPJ_BOOL opj_pi_next_rpcl(opj_pi_iterator_t * pi)
                     }
                     res = &comp->resolutions[pi->resno];
                     levelno = comp->numresolutions - 1 - pi->resno;
-                    /* Avoids division by zero */
-                    /* Relates to id_000004,sig_06,src_000679,op_arith8,pos_49,val_-17 */
-                    /* of  https://github.com/uclouvain/openjpeg/issues/938 */
-                    if (levelno >= 32 ||
-                            ((comp->dx << levelno) >> levelno) != comp->dx ||
-                            ((comp->dy << levelno) >> levelno) != comp->dy) {
-                        continue;
-                    }
-                    if ((comp->dx << levelno) > INT_MAX ||
-                            (comp->dy << levelno) > INT_MAX) {
+
+                    if ((OPJ_UINT32)(((OPJ_UINT64)comp->dx << levelno) >> levelno) != comp->dx ||
+                            (OPJ_UINT32)(((OPJ_UINT64)comp->dy << levelno) >> levelno) != comp->dy) {
                         continue;
                     }
-                    trx0 = opj_uint_ceildiv(pi->tx0, (comp->dx << levelno));
-                    try0 = opj_uint_ceildiv(pi->ty0, (comp->dy << levelno));
-                    trx1 = opj_uint_ceildiv(pi->tx1, (comp->dx << levelno));
-                    try1 = opj_uint_ceildiv(pi->ty1, (comp->dy << levelno));
+
+                    trx0 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->tx0,
+                                                         ((OPJ_UINT64)comp->dx << levelno));
+                    try0 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->ty0,
+                                                         ((OPJ_UINT64)comp->dy << levelno));
+                    trx1 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->tx1,
+                                                         ((OPJ_UINT64)comp->dx << levelno));
+                    try1 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->ty1,
+                                                         ((OPJ_UINT64)comp->dy << levelno));
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;
 
-                    /* To avoid divisions by zero / undefined behaviour on shift */
-                    /* in below tests */
-                    /* Fixes reading id:000026,sig:08,src:002419,op:int32,pos:60,val:+32 */
-                    /* of https://github.com/uclouvain/openjpeg/issues/938 */
-                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
-                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
+                    if ((OPJ_UINT32)(((OPJ_UINT64)comp->dx << rpx) >> rpx) != comp->dx ||
+                            (OPJ_UINT32)(((OPJ_UINT64)comp->dy << rpy) >> rpy) != comp->dy) {
                         continue;
                     }
 
                     /* See ISO-15441. B.12.1.3 Resolution level-position-component-layer progression */
-                    if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
-                            ((try0 << levelno) % (1U << rpy))))) {
+                    if (!(((OPJ_UINT64)pi->y % ((OPJ_UINT64)comp->dy << rpy) == 0) ||
+                            ((pi->y == pi->ty0) &&
+                             (((OPJ_UINT64)try0 << levelno) % ((OPJ_UINT64)1U << rpy))))) {
                         continue;
                     }
-                    if (!((pi->x % (comp->dx << rpx) == 0) || ((pi->x == pi->tx0) &&
-                            ((trx0 << levelno) % (1U << rpx))))) {
+                    if (!(((OPJ_UINT64)pi->x % ((OPJ_UINT64)comp->dx << rpx) == 0) ||
+                            ((pi->x == pi->tx0) &&
+                             (((OPJ_UINT64)trx0 << levelno) % ((OPJ_UINT64)1U << rpx))))) {
                         continue;
                     }
 
@@ -457,11 +453,11 @@ static OPJ_BOOL opj_pi_next_rpcl(opj_pi_iterator_t * pi)
                         continue;
                     }
 
-                    prci = opj_uint_floordivpow2(opj_uint_ceildiv(pi->x,
-                                                 (comp->dx << levelno)), res->pdx)
+                    prci = opj_uint_floordivpow2(opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->x,
+                                                 ((OPJ_UINT64)comp->dx << levelno)), res->pdx)
                            - opj_uint_floordivpow2(trx0, res->pdx);
-                    prcj = opj_uint_floordivpow2(opj_uint_ceildiv(pi->y,
-                                                 (comp->dy << levelno)), res->pdy)
+                    prcj = opj_uint_floordivpow2(opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->y,
+                                                 ((OPJ_UINT64)comp->dy << levelno)), res->pdy)
                            - opj_uint_floordivpow2(try0, res->pdy);
                     pi->precno = prci + prcj * res->pw;
                     for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
@@ -549,41 +545,37 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi)
                     OPJ_UINT32 prci, prcj;
                     res = &comp->resolutions[pi->resno];
                     levelno = comp->numresolutions - 1 - pi->resno;
-                    /* Avoids division by zero */
-                    /* Relates to id_000004,sig_06,src_000679,op_arith8,pos_49,val_-17 */
-                    /* of  https://github.com/uclouvain/openjpeg/issues/938 */
-                    if (levelno >= 32 ||
-                            ((comp->dx << levelno) >> levelno) != comp->dx ||
-                            ((comp->dy << levelno) >> levelno) != comp->dy) {
-                        continue;
-                    }
-                    if ((comp->dx << levelno) > INT_MAX ||
-                            (comp->dy << levelno) > INT_MAX) {
+
+                    if ((OPJ_UINT32)(((OPJ_UINT64)comp->dx << levelno) >> levelno) != comp->dx ||
+                            (OPJ_UINT32)(((OPJ_UINT64)comp->dy << levelno) >> levelno) != comp->dy) {
                         continue;
                     }
-                    trx0 = opj_uint_ceildiv(pi->tx0, (comp->dx << levelno));
-                    try0 = opj_uint_ceildiv(pi->ty0, (comp->dy << levelno));
-                    trx1 = opj_uint_ceildiv(pi->tx1, (comp->dx << levelno));
-                    try1 = opj_uint_ceildiv(pi->ty1, (comp->dy << levelno));
+
+                    trx0 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->tx0,
+                                                         ((OPJ_UINT64)comp->dx << levelno));
+                    try0 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->ty0,
+                                                         ((OPJ_UINT64)comp->dy << levelno));
+                    trx1 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->tx1,
+                                                         ((OPJ_UINT64)comp->dx << levelno));
+                    try1 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->ty1,
+                                                         ((OPJ_UINT64)comp->dy << levelno));
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;
 
-                    /* To avoid divisions by zero / undefined behaviour on shift */
-                    /* in below tests */
-                    /* Relates to id:000019,sig:08,src:001098,op:flip1,pos:49 */
-                    /* of https://github.com/uclouvain/openjpeg/issues/938 */
-                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
-                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
+                    if ((OPJ_UINT32)(((OPJ_UINT64)comp->dx << rpx) >> rpx) != comp->dx ||
+                            (OPJ_UINT32)(((OPJ_UINT64)comp->dy << rpy) >> rpy) != comp->dy) {
                         continue;
                     }
 
                     /* See ISO-15441. B.12.1.4 Position-component-resolution level-layer progression */
-                    if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
-                            ((try0 << levelno) % (1U << rpy))))) {
+                    if (!(((OPJ_UINT64)pi->y % ((OPJ_UINT64)comp->dy << rpy) == 0) ||
+                            ((pi->y == pi->ty0) &&
+                             (((OPJ_UINT64)try0 << levelno) % ((OPJ_UINT64)1U << rpy))))) {
                         continue;
                     }
-                    if (!((pi->x % (comp->dx << rpx) == 0) || ((pi->x == pi->tx0) &&
-                            ((trx0 << levelno) % (1U << rpx))))) {
+                    if (!(((OPJ_UINT64)pi->x % ((OPJ_UINT64)comp->dx << rpx) == 0) ||
+                            ((pi->x == pi->tx0) &&
+                             (((OPJ_UINT64)trx0 << levelno) % ((OPJ_UINT64)1U << rpx))))) {
                         continue;
                     }
 
@@ -595,11 +587,11 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi)
                         continue;
                     }
 
-                    prci = opj_uint_floordivpow2(opj_uint_ceildiv(pi->x,
-                                                 (comp->dx << levelno)), res->pdx)
+                    prci = opj_uint_floordivpow2(opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->x,
+                                                 ((OPJ_UINT64)comp->dx << levelno)), res->pdx)
                            - opj_uint_floordivpow2(trx0, res->pdx);
-                    prcj = opj_uint_floordivpow2(opj_uint_ceildiv(pi->y,
-                                                 (comp->dy << levelno)), res->pdy)
+                    prcj = opj_uint_floordivpow2(opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->y,
+                                                 ((OPJ_UINT64)comp->dy << levelno)), res->pdy)
                            - opj_uint_floordivpow2(try0, res->pdy);
                     pi->precno = prci + prcj * res->pw;
                     for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
@@ -685,40 +677,37 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi)
                     OPJ_UINT32 prci, prcj;
                     res = &comp->resolutions[pi->resno];
                     levelno = comp->numresolutions - 1 - pi->resno;
-                    /* Avoids division by zero on id_000004,sig_06,src_000679,op_arith8,pos_49,val_-17 */
-                    /* of  https://github.com/uclouvain/openjpeg/issues/938 */
-                    if (levelno >= 32 ||
-                            ((comp->dx << levelno) >> levelno) != comp->dx ||
-                            ((comp->dy << levelno) >> levelno) != comp->dy) {
-                        continue;
-                    }
-                    if ((comp->dx << levelno) > INT_MAX ||
-                            (comp->dy << levelno) > INT_MAX) {
+
+                    if ((OPJ_UINT32)(((OPJ_UINT64)comp->dx << levelno) >> levelno) != comp->dx ||
+                            (OPJ_UINT32)(((OPJ_UINT64)comp->dy << levelno) >> levelno) != comp->dy) {
                         continue;
                     }
-                    trx0 = opj_uint_ceildiv(pi->tx0, (comp->dx << levelno));
-                    try0 = opj_uint_ceildiv(pi->ty0, (comp->dy << levelno));
-                    trx1 = opj_uint_ceildiv(pi->tx1, (comp->dx << levelno));
-                    try1 = opj_uint_ceildiv(pi->ty1, (comp->dy << levelno));
+
+                    trx0 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->tx0,
+                                                         ((OPJ_UINT64)comp->dx << levelno));
+                    try0 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->ty0,
+                                                         ((OPJ_UINT64)comp->dy << levelno));
+                    trx1 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->tx1,
+                                                         ((OPJ_UINT64)comp->dx << levelno));
+                    try1 = opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->ty1,
+                                                         ((OPJ_UINT64)comp->dy << levelno));
                     rpx = res->pdx + levelno;
                     rpy = res->pdy + levelno;
 
-                    /* To avoid divisions by zero / undefined behaviour on shift */
-                    /* in below tests */
-                    /* Fixes reading id:000019,sig:08,src:001098,op:flip1,pos:49 */
-                    /* of https://github.com/uclouvain/openjpeg/issues/938 */
-                    if (rpx >= 31 || ((comp->dx << rpx) >> rpx) != comp->dx ||
-                            rpy >= 31 || ((comp->dy << rpy) >> rpy) != comp->dy) {
+                    if ((OPJ_UINT32)(((OPJ_UINT64)comp->dx << rpx) >> rpx) != comp->dx ||
+                            (OPJ_UINT32)(((OPJ_UINT64)comp->dy << rpy) >> rpy) != comp->dy) {
                         continue;
                     }
 
                     /* See ISO-15441. B.12.1.5 Component-position-resolution level-layer progression */
-                    if (!((pi->y % (comp->dy << rpy) == 0) || ((pi->y == pi->ty0) &&
-                            ((try0 << levelno) % (1U << rpy))))) {
+                    if (!(((OPJ_UINT64)pi->y % ((OPJ_UINT64)comp->dy << rpy) == 0) ||
+                            ((pi->y == pi->ty0) &&
+                             (((OPJ_UINT64)try0 << levelno) % ((OPJ_UINT64)1U << rpy))))) {
                         continue;
                     }
-                    if (!((pi->x % (comp->dx << rpx) == 0) || ((pi->x == pi->tx0) &&
-                            ((trx0 << levelno) % (1U << rpx))))) {
+                    if (!(((OPJ_UINT64)pi->x % ((OPJ_UINT64)comp->dx << rpx) == 0) ||
+                            ((pi->x == pi->tx0) &&
+                             (((OPJ_UINT64)trx0 << levelno) % ((OPJ_UINT64)1U << rpx))))) {
                         continue;
                     }
 
@@ -730,11 +719,11 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi)
                         continue;
                     }
 
-                    prci = opj_uint_floordivpow2(opj_uint_ceildiv(pi->x,
-                                                 (comp->dx << levelno)), res->pdx)
+                    prci = opj_uint_floordivpow2(opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->x,
+                                                 ((OPJ_UINT64)comp->dx << levelno)), res->pdx)
                            - opj_uint_floordivpow2(trx0, res->pdx);
-                    prcj = opj_uint_floordivpow2(opj_uint_ceildiv(pi->y,
-                                                 (comp->dy << levelno)), res->pdy)
+                    prcj = opj_uint_floordivpow2(opj_uint64_ceildiv_res_uint32((OPJ_UINT64)pi->y,
+                                                 ((OPJ_UINT64)comp->dy << levelno)), res->pdy)
                            - opj_uint_floordivpow2(try0, res->pdy);
                     pi->precno = (OPJ_UINT32)(prci + prcj * res->pw);
                     for (pi->layno = pi->poc.layno0; pi->layno < pi->poc.layno1; pi->layno++) {
@@ -837,18 +826,24 @@ static void opj_get_encoding_parameters(const opj_image_t *p_image,
 
         /* use custom size for precincts */
         for (resno = 0; resno < l_tccp->numresolutions; ++resno) {
-            OPJ_UINT32 l_dx, l_dy;
+            OPJ_UINT64 l_dx, l_dy;
 
             /* precinct width and height */
             l_pdx = l_tccp->prcw[resno];
             l_pdy = l_tccp->prch[resno];
 
-            l_dx = l_img_comp->dx * (1u << (l_pdx + l_tccp->numresolutions - 1 - resno));
-            l_dy = l_img_comp->dy * (1u << (l_pdy + l_tccp->numresolutions - 1 - resno));
+            l_dx = l_img_comp->dx * ((OPJ_UINT64)1u << (l_pdx + l_tccp->numresolutions - 1 -
+                                     resno));
+            l_dy = l_img_comp->dy * ((OPJ_UINT64)1u << (l_pdy + l_tccp->numresolutions - 1 -
+                                     resno));
 
             /* take the minimum size for dx for each comp and resolution */
-            *p_dx_min = opj_uint_min(*p_dx_min, l_dx);
-            *p_dy_min = opj_uint_min(*p_dy_min, l_dy);
+            if (l_dx <= UINT_MAX) {
+                *p_dx_min = opj_uint_min(*p_dx_min, (OPJ_UINT32)l_dx);
+            }
+            if (l_dy <= UINT_MAX) {
+                *p_dy_min = opj_uint_min(*p_dy_min, (OPJ_UINT32)l_dy);
+            }
 
             /* various calculations of extents */
             l_level_no = l_tccp->numresolutions - 1 - resno;