opj_pi_next_rpcl / opj_pi_next_pcrl / opj_pi_next_cprl: avoid int overflow (#895)

Fixes int overflow on openjeg-crashes-2017-07-27/id:000000,sig:08,src:000879,op:flip2,pos:128.jp2

diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c
index 66af35f4553a721cbd3d728baa4fcbcc94cff213..4e2e48ad4acd4ece33f15af51c5a4f7b61f07862 100644 (file)
--- a/src/lib/openjp2/pi.c
+++ b/src/lib/openjp2/pi.c
@@ -400,6 +400,10 @@ static OPJ_BOOL opj_pi_next_rpcl(opj_pi_iterator_t * pi)
                             ((comp->dy << levelno) >> levelno) != comp->dy) {
                         continue;
                     }
+                    if ((comp->dx << levelno) > INT_MAX ||
+                            (comp->dy << levelno) > INT_MAX) {
+                        continue;
+                    }
                     trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
                     try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
                     trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
@@ -526,6 +530,10 @@ static OPJ_BOOL opj_pi_next_pcrl(opj_pi_iterator_t * pi)
                             ((comp->dy << levelno) >> levelno) != comp->dy) {
                         continue;
                     }
+                    if ((comp->dx << levelno) > INT_MAX ||
+                            (comp->dy << levelno) > INT_MAX) {
+                        continue;
+                    }
                     trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
                     try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
                     trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));
@@ -650,6 +658,10 @@ static OPJ_BOOL opj_pi_next_cprl(opj_pi_iterator_t * pi)
                             ((comp->dy << levelno) >> levelno) != comp->dy) {
                         continue;
                     }
+                    if ((comp->dx << levelno) > INT_MAX ||
+                            (comp->dy << levelno) > INT_MAX) {
+                        continue;
+                    }
                     trx0 = opj_int_ceildiv(pi->tx0, (OPJ_INT32)(comp->dx << levelno));
                     try0 = opj_int_ceildiv(pi->ty0, (OPJ_INT32)(comp->dy << levelno));
                     trx1 = opj_int_ceildiv(pi->tx1, (OPJ_INT32)(comp->dx << levelno));