Remove limitation that prevents from opening images bigger than 4 billion pixels

However the intermediate buffer for decoding must still be smaller than 4
billion pixels, so this is useful for decoding at a lower resolution level,
or subtile decoding.

2 files changed, 44 insertions, 41 deletions

diff --git a/src/lib/openjp2/j2k.c b/src/lib/openjp2/j2k.c
index 0d8bbc3f..400fcaa4 100644
--- a/src/lib/openjp2/j2k.c
+++ b/src/lib/openjp2/j2k.c
@@ -2142,13 +2142,6 @@ static OPJ_BOOL opj_j2k_read_siz(opj_j2k_t *p_j2k,
         return OPJ_FALSE;
     }
 
-    /* testcase 1610.pdf.SIGSEGV.59c.681 */
-    if ((0xFFFFFFFFU / l_image->x1) < l_image->y1) {
-        opj_event_msg(p_manager, EVT_ERROR,
-                      "Prevent buffer overflow (x1: %d, y1: %d)\n", l_image->x1, l_image->y1);
-        return OPJ_FALSE;
-    }
-
     /* testcase issue427-illegal-tile-offset.jp2 */
     l_tx1 = opj_uint_adds(l_cp->tx0, l_cp->tdx); /* manage overflow */
     l_ty1 = opj_uint_adds(l_cp->ty0, l_cp->tdy); /* manage overflow */
@@ -8787,9 +8780,13 @@ OPJ_BOOL opj_j2k_read_tile_header(opj_j2k_t * p_j2k,
 
     *p_tile_index = p_j2k->m_current_tile_number;
     *p_go_on = OPJ_TRUE;
-    *p_data_size = opj_tcd_get_decoded_tile_size(p_j2k->m_tcd, OPJ_FALSE);
-    if (*p_data_size == UINT_MAX) {
-        return OPJ_FALSE;
+    if (p_data_size) {
+        /* For internal use in j2k.c, we don't need this */
+        /* This is just needed for folks using the opj_read_tile_header() / opj_decode_tile_data() combo */
+        *p_data_size = opj_tcd_get_decoded_tile_size(p_j2k->m_tcd, OPJ_FALSE);
+        if (*p_data_size == UINT_MAX) {
+            return OPJ_FALSE;
+        }
     }
     *p_tile_x0 = p_j2k->m_tcd->tcd_image->tiles->x0;
     *p_tile_y0 = p_j2k->m_tcd->tcd_image->tiles->y0;
@@ -10466,7 +10463,6 @@ static OPJ_BOOL opj_j2k_decode_tiles(opj_j2k_t *p_j2k,
 {
     OPJ_BOOL l_go_on = OPJ_TRUE;
     OPJ_UINT32 l_current_tile_no;
-    OPJ_UINT32 l_data_size;
     OPJ_INT32 l_tile_x0, l_tile_y0, l_tile_x1, l_tile_y1;
     OPJ_UINT32 l_nb_comps;
     OPJ_UINT32 nr_tiles = 0;
@@ -10483,7 +10479,7 @@ static OPJ_BOOL opj_j2k_decode_tiles(opj_j2k_t *p_j2k,
         OPJ_UINT32 i;
         if (! opj_j2k_read_tile_header(p_j2k,
                                        &l_current_tile_no,
-                                       &l_data_size,
+                                       NULL,
                                        &l_tile_x0, &l_tile_y0,
                                        &l_tile_x1, &l_tile_y1,
                                        &l_nb_comps,
@@ -10515,7 +10511,7 @@ static OPJ_BOOL opj_j2k_decode_tiles(opj_j2k_t *p_j2k,
     for (;;) {
         if (! opj_j2k_read_tile_header(p_j2k,
                                        &l_current_tile_no,
-                                       &l_data_size,
+                                       NULL,
                                        &l_tile_x0, &l_tile_y0,
                                        &l_tile_x1, &l_tile_y1,
                                        &l_nb_comps,
@@ -10589,7 +10585,6 @@ static OPJ_BOOL opj_j2k_decode_one_tile(opj_j2k_t *p_j2k,
     OPJ_BOOL l_go_on = OPJ_TRUE;
     OPJ_UINT32 l_current_tile_no;
     OPJ_UINT32 l_tile_no_to_dec;
-    OPJ_UINT32 l_data_size;
     OPJ_INT32 l_tile_x0, l_tile_y0, l_tile_x1, l_tile_y1;
     OPJ_UINT32 l_nb_comps;
     OPJ_UINT32 l_nb_tiles;
@@ -10640,7 +10635,7 @@ static OPJ_BOOL opj_j2k_decode_one_tile(opj_j2k_t *p_j2k,
     for (;;) {
         if (! opj_j2k_read_tile_header(p_j2k,
                                        &l_current_tile_no,
-                                       &l_data_size,
+                                       NULL,
                                        &l_tile_x0, &l_tile_y0,
                                        &l_tile_x1, &l_tile_y1,
                                        &l_nb_comps,
diff --git a/src/lib/openjp2/tcd.c b/src/lib/openjp2/tcd.c
index 6758487a..2ce55c2d 100644
--- a/src/lib/openjp2/tcd.c
+++ b/src/lib/openjp2/tcd.c
@@ -886,27 +886,6 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
             l_res->x1 = opj_int_ceildivpow2(l_tilec->x1, (OPJ_INT32)l_level_no);
             l_res->y1 = opj_int_ceildivpow2(l_tilec->y1, (OPJ_INT32)l_level_no);
 
-            if (!isEncoder && resno + 1 == l_tilec->minimum_num_resolutions) {
-                /* compute l_data_size with overflow check */
-                OPJ_UINT32 res_w = (OPJ_UINT32)(l_res->x1 - l_res->x0);
-                OPJ_UINT32 res_h = (OPJ_UINT32)(l_res->y1 - l_res->y0);
-
-                /* issue 733, l_data_size == 0U, probably something wrong should be checked before getting here */
-                if (res_h > 0 && res_w > (((OPJ_UINT32) - 1) / res_h)) {
-                    opj_event_msg(manager, EVT_ERROR, "Not enough memory for tile data\n");
-                    return OPJ_FALSE;
-                }
-                l_data_size = res_w * res_h;
-
-                if ((((OPJ_UINT32) - 1) / (OPJ_UINT32)sizeof(OPJ_UINT32)) < l_data_size) {
-                    opj_event_msg(manager, EVT_ERROR, "Not enough memory for tile data\n");
-                    return OPJ_FALSE;
-                }
-                l_data_size *= (OPJ_UINT32)sizeof(OPJ_UINT32);
-
-                l_tilec->data_size_needed = l_data_size;
-            }
-
             /*fprintf(stderr, "\t\t\tres_x0= %d, res_y0 =%d, res_x1=%d, res_y1=%d\n", l_res->x0, l_res->y0, l_res->x1, l_res->y1);*/
             /* p. 35, table A-23, ISO/IEC FDIS154444-1 : 2000 (18 august 2000) */
             l_pdx = l_tccp->prcw[resno];
@@ -1312,6 +1291,7 @@ OPJ_UINT32 opj_tcd_get_decoded_tile_size(opj_tcd_t *p_tcd,
     l_img_comp = p_tcd->image->comps;
 
     for (i = 0; i < p_tcd->image->numcomps; ++i) {
+        OPJ_UINT32 w, h;
         l_size_comp = l_img_comp->prec >> 3; /*(/ 8)*/
         l_remaining = l_img_comp->prec & 7;  /* (%8) */
 
@@ -1325,12 +1305,16 @@ OPJ_UINT32 opj_tcd_get_decoded_tile_size(opj_tcd_t *p_tcd,
 
         l_res = l_tile_comp->resolutions + l_tile_comp->minimum_num_resolutions - 1;
         if (take_into_account_partial_decoding && !p_tcd->whole_tile_decoding) {
-            l_temp = (l_res->win_x1 - l_res->win_x0) *
-                     (l_res->win_y1 - l_res->win_y0);
+            w = l_res->win_x1 - l_res->win_x0;
+            h = l_res->win_y1 - l_res->win_y0;
         } else {
-            l_temp = (OPJ_UINT32)((l_res->x1 - l_res->x0) * (l_res->y1 -
-                                  l_res->y0)); /* x1*y1 can't overflow */
+            w = (OPJ_UINT32)(l_res->x1 - l_res->x0);
+            h = (OPJ_UINT32)(l_res->y1 - l_res->y0);
+        }
+        if (h > 0 && UINT_MAX / w < h) {
+            return UINT_MAX;
         }
+        l_temp = w * h;
         if (l_size_comp && UINT_MAX / l_size_comp < l_temp) {
             return UINT_MAX;
         }
@@ -1473,7 +1457,31 @@ OPJ_BOOL opj_tcd_decode_tile(opj_tcd_t *p_tcd,
 
     if (p_tcd->whole_tile_decoding) {
         for (compno = 0; compno < p_tcd->image->numcomps; compno++) {
-            if (!opj_alloc_tile_component_data(&(p_tcd->tcd_image->tiles->comps[compno]))) {
+            opj_tcd_tilecomp_t* tilec = &(p_tcd->tcd_image->tiles->comps[compno]);
+            opj_tcd_resolution_t *l_res = &
+                                          (tilec->resolutions[tilec->minimum_num_resolutions - 1]);
+            OPJ_UINT32 l_data_size;
+
+            /* compute l_data_size with overflow check */
+            OPJ_UINT32 res_w = (OPJ_UINT32)(l_res->x1 - l_res->x0);
+            OPJ_UINT32 res_h = (OPJ_UINT32)(l_res->y1 - l_res->y0);
+
+            /* issue 733, l_data_size == 0U, probably something wrong should be checked before getting here */
+            if (res_h > 0 && res_w > (((OPJ_UINT32) - 1) / res_h)) {
+                opj_event_msg(p_manager, EVT_ERROR, "Not enough memory for tile data\n");
+                return OPJ_FALSE;
+            }
+            l_data_size = res_w * res_h;
+
+            if ((((OPJ_UINT32) - 1) / (OPJ_UINT32)sizeof(OPJ_UINT32)) < l_data_size) {
+                opj_event_msg(p_manager, EVT_ERROR, "Not enough memory for tile data\n");
+                return OPJ_FALSE;
+            }
+            l_data_size *= (OPJ_UINT32)sizeof(OPJ_UINT32);
+
+            tilec->data_size_needed = l_data_size;
+
+            if (!opj_alloc_tile_component_data(tilec)) {
                 opj_event_msg(p_manager, EVT_ERROR, "Not enough memory for tile data\n");
                 return OPJ_FALSE;
             }