Cleanup code related to quality layer allocation, and add a few safety checks

[openjpeg.git] / src / lib / openjp2 / t1.c

diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index f5fd233917d2e01790efb4cb9fa42dc09677fe29..52e466eb974afd1fc89f0ed180b95973ee8bc532 100644 (file)
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -1410,7 +1410,6 @@ static void opj_t1_dec_clnpass(
 }
 
 
-/** mod fixed_quality */
 static OPJ_FLOAT64 opj_t1_getwmsedec(
     OPJ_INT32 nmsedec,
     OPJ_UINT32 compno,
@@ -2313,7 +2312,7 @@ OPJ_BOOL opj_t1_encode_cblks(opj_tcd_t* tcd,
     OPJ_UINT32 compno, resno, bandno, precno, cblkno;
     opj_mutex_t* mutex = opj_mutex_create();
 
-    tile->distotile = 0;        /* fixed_quality */
+    tile->distotile = 0;
 
     for (compno = 0; compno < tile->numcomps; ++compno) {
         opj_tcd_tilecomp_t* tilec = &tile->comps[compno];
@@ -2401,7 +2400,6 @@ static int opj_t1_enc_is_term_pass(opj_tcd_cblk_enc_t* cblk,
 }
 
 
-/** mod fixed_quality */
 static OPJ_FLOAT64 opj_t1_encode_cblk(opj_t1_t *t1,
                                       opj_tcd_cblk_enc_t* cblk,
                                       OPJ_UINT32 orient,
@@ -2443,6 +2441,13 @@ static OPJ_FLOAT64 opj_t1_encode_cblk(opj_t1_t *t1,
             OPJ_INT32 tmp = *datap;
             if (tmp < 0) {
                 OPJ_UINT32 tmp_unsigned;
+                if (tmp == INT_MIN) {
+                    /* To avoid undefined behaviour when negating INT_MIN */
+                    /* but if we go here, it means we have supplied an input */
+                    /* with more bit depth than we we can really support. */
+                    /* Cf https://github.com/uclouvain/openjpeg/issues/1432 */
+                    tmp = INT_MIN + 1;
+                }
                 max = opj_int_max(max, -tmp);
                 tmp_unsigned = opj_to_smr(tmp);
                 memcpy(datap, &tmp_unsigned, sizeof(OPJ_INT32));
@@ -2498,7 +2503,6 @@ static OPJ_FLOAT64 opj_t1_encode_cblk(opj_t1_t *t1,
             break;
         }
 
-        /* fixed_quality */
         tempwmsedec = opj_t1_getwmsedec(nmsedec, compno, level, orient, bpno, qmfbid,
                                         stepsize, numcomps, mct_norms, mct_numcomps) ;
         cumwmsedec += tempwmsedec;