Encoder: fix packet writing of empty sub-bands (#891, #892)

There are situations where, given a tile size, at a resolution level,
there are sub-bands with x0==x1 or y0==y1, that consequently don't have any
valid codeblocks, but the other sub-bands may be non-empty.
Given that we recycle the memory from one tile to another one, those
ghost codeblocks might be non-0 and thus candidate for packet inclusion.

4 files changed, 71 insertions, 20 deletions

diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index 15ac0407..94aa9b29 100644
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -2008,8 +2008,14 @@ OPJ_BOOL opj_t1_encode_cblks(opj_t1_t *t1,
 
             for (bandno = 0; bandno < res->numbands; ++bandno) {
                 opj_tcd_band_t* OPJ_RESTRICT band = &res->bands[bandno];
-                OPJ_INT32 bandconst = 8192 * 8192 / ((OPJ_INT32) floor(band->stepsize * 8192));
+                OPJ_INT32 bandconst;
 
+                /* Skip empty bands */
+                if (opj_tcd_is_band_empty(band)) {
+                    continue;
+                }
+
+                bandconst = 8192 * 8192 / ((OPJ_INT32) floor(band->stepsize * 8192));
                 for (precno = 0; precno < res->pw * res->ph; ++precno) {
                     opj_tcd_precinct_t *prc = &band->precincts[precno];
 
diff --git a/src/lib/openjp2/t2.c b/src/lib/openjp2/t2.c
index b0990963..f7395dfc 100644
--- a/src/lib/openjp2/t2.c
+++ b/src/lib/openjp2/t2.c
@@ -633,9 +633,15 @@ static OPJ_BOOL opj_t2_encode_packet(OPJ_UINT32 tileno,
     if (!layno) {
         band = res->bands;
 
-        for (bandno = 0; bandno < res->numbands; ++bandno) {
-            opj_tcd_precinct_t *prc = &band->precincts[precno];
+        for (bandno = 0; bandno < res->numbands; ++bandno, ++band) {
+            opj_tcd_precinct_t *prc;
 
+            /* Skip empty bands */
+            if (opj_tcd_is_band_empty(band)) {
+                continue;
+            }
+
+            prc = &band->precincts[precno];
             opj_tgt_reset(prc->incltree);
             opj_tgt_reset(prc->imsbtree);
 
@@ -646,7 +652,6 @@ static OPJ_BOOL opj_t2_encode_packet(OPJ_UINT32 tileno,
                 cblk->numpasses = 0;
                 opj_tgt_setvalue(prc->imsbtree, cblkno, band->numbps - (OPJ_INT32)cblk->numbps);
             }
-            ++band;
         }
     }
 
@@ -660,9 +665,15 @@ static OPJ_BOOL opj_t2_encode_packet(OPJ_UINT32 tileno,
 
     /* Writing Packet header */
     band = res->bands;
-    for (bandno = 0; bandno < res->numbands; ++bandno)      {
-        opj_tcd_precinct_t *prc = &band->precincts[precno];
+    for (bandno = 0; bandno < res->numbands; ++bandno, ++band)      {
+        opj_tcd_precinct_t *prc;
+
+        /* Skip empty bands */
+        if (opj_tcd_is_band_empty(band)) {
+            continue;
+        }
 
+        prc = &band->precincts[precno];
         l_nb_blocks = prc->cw * prc->ch;
         cblk = prc->cblks.enc;
 
@@ -745,8 +756,6 @@ static OPJ_BOOL opj_t2_encode_packet(OPJ_UINT32 tileno,
 
             ++cblk;
         }
-
-        ++band;
     }
 
     if (!opj_bio_flush(bio)) {
@@ -780,9 +789,15 @@ static OPJ_BOOL opj_t2_encode_packet(OPJ_UINT32 tileno,
 
     /* Writing the packet body */
     band = res->bands;
-    for (bandno = 0; bandno < res->numbands; bandno++) {
-        opj_tcd_precinct_t *prc = &band->precincts[precno];
+    for (bandno = 0; bandno < res->numbands; bandno++, ++band) {
+        opj_tcd_precinct_t *prc;
 
+        /* Skip empty bands */
+        if (opj_tcd_is_band_empty(band)) {
+            continue;
+        }
+
+        prc = &band->precincts[precno];
         l_nb_blocks = prc->cw * prc->ch;
         cblk = prc->cblks.enc;
 
@@ -815,7 +830,6 @@ static OPJ_BOOL opj_t2_encode_packet(OPJ_UINT32 tileno,
             ++cblk;
             /* INDEX >> */
         }
-        ++band;
     }
 
     assert(c >= dest);
@@ -902,7 +916,7 @@ static OPJ_BOOL opj_t2_read_packet_header(opj_t2_t* p_t2,
 
         /* reset tagtrees */
         for (bandno = 0; bandno < l_res->numbands; ++bandno) {
-            if (!((l_band->x1 - l_band->x0 == 0) || (l_band->y1 - l_band->y0 == 0))) {
+            if (!opj_tcd_is_band_empty(l_band)) {
                 opj_tcd_precinct_t *l_prc = &l_band->precincts[p_pi->precno];
                 if (!(p_pi->precno < (l_band->precincts_data_size / sizeof(
                                           opj_tcd_precinct_t)))) {
@@ -1011,11 +1025,10 @@ static OPJ_BOOL opj_t2_read_packet_header(opj_t2_t* p_t2,
     }
 
     l_band = l_res->bands;
-    for (bandno = 0; bandno < l_res->numbands; ++bandno) {
+    for (bandno = 0; bandno < l_res->numbands; ++bandno, ++l_band) {
         opj_tcd_precinct_t *l_prc = &(l_band->precincts[p_pi->precno]);
 
-        if ((l_band->x1 - l_band->x0 == 0) || (l_band->y1 - l_band->y0 == 0)) {
-            ++l_band;
+        if (opj_tcd_is_band_empty(l_band)) {
             continue;
         }
 
@@ -1101,8 +1114,6 @@ static OPJ_BOOL opj_t2_read_packet_header(opj_t2_t* p_t2,
 
             ++l_cblk;
         }
-
-        ++l_band;
     }
 
     if (!opj_bio_inalign(l_bio)) {
diff --git a/src/lib/openjp2/tcd.c b/src/lib/openjp2/tcd.c
index e36496e6..70bb9d7b 100644
--- a/src/lib/openjp2/tcd.c
+++ b/src/lib/openjp2/tcd.c
@@ -245,6 +245,11 @@ void opj_tcd_makelayer(opj_tcd_t *tcd,
             for (bandno = 0; bandno < res->numbands; bandno++) {
                 opj_tcd_band_t *band = &res->bands[bandno];
 
+                /* Skip empty bands */
+                if (opj_tcd_is_band_empty(band)) {
+                    continue;
+                }
+
                 for (precno = 0; precno < res->pw * res->ph; precno++) {
                     opj_tcd_precinct_t *prc = &band->precincts[precno];
 
@@ -347,6 +352,11 @@ void opj_tcd_makelayer_fixed(opj_tcd_t *tcd, OPJ_UINT32 layno,
             for (bandno = 0; bandno < res->numbands; bandno++) {
                 opj_tcd_band_t *band = &res->bands[bandno];
 
+                /* Skip empty bands */
+                if (opj_tcd_is_band_empty(band)) {
+                    continue;
+                }
+
                 for (precno = 0; precno < res->pw * res->ph; precno++) {
                     opj_tcd_precinct_t *prc = &band->precincts[precno];
 
@@ -447,6 +457,11 @@ OPJ_BOOL opj_tcd_rateallocate(opj_tcd_t *tcd,
             for (bandno = 0; bandno < res->numbands; bandno++) {
                 opj_tcd_band_t *band = &res->bands[bandno];
 
+                /* Skip empty bands */
+                if (opj_tcd_is_band_empty(band)) {
+                    continue;
+                }
+
                 for (precno = 0; precno < res->pw * res->ph; precno++) {
                     opj_tcd_precinct_t *prc = &band->precincts[precno];
 
@@ -906,7 +921,7 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
             cblkheightexpn = opj_uint_min(l_tccp->cblkh, cbgheightexpn);
             l_band = l_res->bands;
 
-            for (bandno = 0; bandno < l_res->numbands; ++bandno) {
+            for (bandno = 0; bandno < l_res->numbands; ++bandno, ++l_band, ++l_step_size) {
                 OPJ_INT32 numbps;
                 /*fprintf(stderr, "\t\t\tband_no=%d/%d\n", bandno, l_res->numbands );*/
 
@@ -933,6 +948,16 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                                                        l_level_no), (OPJ_INT32)(l_level_no + 1));
                 }
 
+                if (isEncoder) {
+                    /* Skip empty bands */
+                    if (opj_tcd_is_band_empty(l_band)) {
+                        /* Do not zero l_band->precints to avoid leaks */
+                        /* but make sure we don't use it later, since */
+                        /* it will point to precincts of previous bands... */
+                        continue;
+                    }
+                }
+
                 /** avoid an if with storing function pointer */
                 l_gain = (*l_gain_ptr)(l_band->bandno);
                 numbps = (OPJ_INT32)(l_image_comp->prec + l_gain);
@@ -1099,8 +1124,6 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                     }
                     ++l_current_precinct;
                 } /* precno */
-                ++l_band;
-                ++l_step_size;
             } /* bandno */
             ++l_res;
         } /* resno */
@@ -2280,3 +2303,8 @@ OPJ_BOOL opj_tcd_copy_tile_data(opj_tcd_t *p_tcd,
 
     return OPJ_TRUE;
 }
+
+OPJ_BOOL opj_tcd_is_band_empty(opj_tcd_band_t* band)
+{
+    return (band->x1 - band->x0 == 0) || (band->y1 - band->y0 == 0);
+}
diff --git a/src/lib/openjp2/tcd.h b/src/lib/openjp2/tcd.h
index 913979a8..4f842b41 100644
--- a/src/lib/openjp2/tcd.h
+++ b/src/lib/openjp2/tcd.h
@@ -375,6 +375,12 @@ OPJ_BOOL opj_tcd_copy_tile_data(opj_tcd_t *p_tcd,
  */
 OPJ_BOOL opj_alloc_tile_component_data(opj_tcd_tilecomp_t *l_tilec);
 
+/** Returns whether a sub-band is empty (i.e. whether it has a null area)
+ * @param band Sub-band handle.
+ * @return OPJ_TRUE whether the sub-band is empty.
+ */
+OPJ_BOOL opj_tcd_is_band_empty(opj_tcd_band_t* band);
+
 /* ----------------------------------------------------------------------- */
 /*@}*/