opj_tcd_get_decoded_tile_size(): fix potential UINT32 overflow (#854, CVE-2016-5152)

Fix derived from https://pdfium.googlesource.com/pdfium.git/+/d8cc503575463ff3d81b22dad292665f2c88911e/third_party/libopenjpeg20/0018-tcd_get_decoded_tile_size.patch

diff --git a/src/lib/openjp2/j2k.c b/src/lib/openjp2/j2k.c
index 3d90de1ef82a690027f9a059b6be8a9021694264..3a8f08ea2b4de44510194ef12448148bd0d5d205 100644 (file)
--- a/src/lib/openjp2/j2k.c
+++ b/src/lib/openjp2/j2k.c
@@ -8655,6 +8655,9 @@ OPJ_BOOL opj_j2k_read_tile_header(opj_j2k_t * p_j2k,
     *p_tile_index = p_j2k->m_current_tile_number;
     *p_go_on = OPJ_TRUE;
     *p_data_size = opj_tcd_get_decoded_tile_size(p_j2k->m_tcd);
+    if (*p_data_size == UINT_MAX) {
+        return OPJ_FALSE;
+    }
     *p_tile_x0 = p_j2k->m_tcd->tcd_image->tiles->x0;
     *p_tile_y0 = p_j2k->m_tcd->tcd_image->tiles->y0;
     *p_tile_x1 = p_j2k->m_tcd->tcd_image->tiles->x1;

diff --git a/src/lib/openjp2/tcd.c b/src/lib/openjp2/tcd.c
index 4d53e8b5e9ed8a4b6b44c6a21eca3ccd43ff102a..567798525be9b93c62488cef77badf6a743c7e7d 100644 (file)
--- a/src/lib/openjp2/tcd.c
+++ b/src/lib/openjp2/tcd.c
@@ -1256,6 +1256,7 @@ OPJ_UINT32 opj_tcd_get_decoded_tile_size(opj_tcd_t *p_tcd)
     opj_tcd_tilecomp_t * l_tile_comp = 00;
     opj_tcd_resolution_t * l_res = 00;
     OPJ_UINT32 l_size_comp, l_remaining;
+    OPJ_UINT32 l_temp;
 
     l_tile_comp = p_tcd->tcd_image->tiles->comps;
     l_img_comp = p_tcd->image->comps;
@@ -1273,8 +1274,17 @@ OPJ_UINT32 opj_tcd_get_decoded_tile_size(opj_tcd_t *p_tcd)
         }
 
         l_res = l_tile_comp->resolutions + l_tile_comp->minimum_num_resolutions - 1;
-        l_data_size += l_size_comp * (OPJ_UINT32)((l_res->x1 - l_res->x0) *
-                       (l_res->y1 - l_res->y0));
+        l_temp = (OPJ_UINT32)((l_res->x1 - l_res->x0) * (l_res->y1 -
+                              l_res->y0)); /* x1*y1 can't overflow */
+        if (l_size_comp && UINT_MAX / l_size_comp < l_temp) {
+            return UINT_MAX;
+        }
+        l_temp *= l_size_comp;
+
+        if (l_temp > UINT_MAX - l_data_size) {
+            return UINT_MAX;
+        }
+        l_data_size += l_temp;
         ++l_img_comp;
         ++l_tile_comp;
     }
@@ -1469,7 +1479,7 @@ OPJ_BOOL opj_tcd_update_tile_data(opj_tcd_t *p_tcd,
     OPJ_UINT32 l_stride, l_width, l_height;
 
     l_data_size = opj_tcd_get_decoded_tile_size(p_tcd);
-    if (l_data_size > p_dest_length) {
+    if (l_data_size == UINT_MAX || l_data_size > p_dest_length) {
         return OPJ_FALSE;
     }