opj_tcd_dc_level_shift_decode(): avoid int overflow. Fixes https://bugs.chromium...
[openjpeg.git] / src / lib / openjp2 / tcd.c
index dc5c89b91259a7fa6a7c093df2e9029a613aac02..4d53e8b5e9ed8a4b6b44c6a21eca3ccd43ff102a 100644 (file)
@@ -157,7 +157,8 @@ static OPJ_BOOL opj_tcd_t2_decode(opj_tcd_t *p_tcd,
                                   opj_codestream_index_t *p_cstr_index,
                                   opj_event_mgr_t *p_manager);
 
-static OPJ_BOOL opj_tcd_t1_decode(opj_tcd_t *p_tcd);
+static OPJ_BOOL opj_tcd_t1_decode(opj_tcd_t *p_tcd,
+                                  opj_event_mgr_t *p_manager);
 
 static OPJ_BOOL opj_tcd_dwt_decode(opj_tcd_t *p_tcd);
 
@@ -246,6 +247,11 @@ void opj_tcd_makelayer(opj_tcd_t *tcd,
             for (bandno = 0; bandno < res->numbands; bandno++) {
                 opj_tcd_band_t *band = &res->bands[bandno];
 
+                /* Skip empty bands */
+                if (opj_tcd_is_band_empty(band)) {
+                    continue;
+                }
+
                 for (precno = 0; precno < res->pw * res->ph; precno++) {
                     opj_tcd_precinct_t *prc = &band->precincts[precno];
 
@@ -348,6 +354,11 @@ void opj_tcd_makelayer_fixed(opj_tcd_t *tcd, OPJ_UINT32 layno,
             for (bandno = 0; bandno < res->numbands; bandno++) {
                 opj_tcd_band_t *band = &res->bands[bandno];
 
+                /* Skip empty bands */
+                if (opj_tcd_is_band_empty(band)) {
+                    continue;
+                }
+
                 for (precno = 0; precno < res->pw * res->ph; precno++) {
                     opj_tcd_precinct_t *prc = &band->precincts[precno];
 
@@ -448,6 +459,11 @@ OPJ_BOOL opj_tcd_rateallocate(opj_tcd_t *tcd,
             for (bandno = 0; bandno < res->numbands; bandno++) {
                 opj_tcd_band_t *band = &res->bands[bandno];
 
+                /* Skip empty bands */
+                if (opj_tcd_is_band_empty(band)) {
+                    continue;
+                }
+
                 for (precno = 0; precno < res->pw * res->ph; precno++) {
                     opj_tcd_precinct_t *prc = &band->precincts[precno];
 
@@ -907,7 +923,7 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
             cblkheightexpn = opj_uint_min(l_tccp->cblkh, cbgheightexpn);
             l_band = l_res->bands;
 
-            for (bandno = 0; bandno < l_res->numbands; ++bandno) {
+            for (bandno = 0; bandno < l_res->numbands; ++bandno, ++l_band, ++l_step_size) {
                 OPJ_INT32 numbps;
                 /*fprintf(stderr, "\t\t\tband_no=%d/%d\n", bandno, l_res->numbands );*/
 
@@ -934,6 +950,16 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                                                        l_level_no), (OPJ_INT32)(l_level_no + 1));
                 }
 
+                if (isEncoder) {
+                    /* Skip empty bands */
+                    if (opj_tcd_is_band_empty(l_band)) {
+                        /* Do not zero l_band->precints to avoid leaks */
+                        /* but make sure we don't use it later, since */
+                        /* it will point to precincts of previous bands... */
+                        continue;
+                    }
+                }
+
                 /** avoid an if with storing function pointer */
                 l_gain = (*l_gain_ptr)(l_band->bandno);
                 numbps = (OPJ_INT32)(l_image_comp->prec + l_gain);
@@ -1100,8 +1126,6 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                     }
                     ++l_current_precinct;
                 } /* precno */
-                ++l_band;
-                ++l_step_size;
             } /* bandno */
             ++l_res;
         } /* resno */
@@ -1163,7 +1187,8 @@ static OPJ_BOOL opj_tcd_code_block_enc_allocate_data(opj_tcd_cblk_enc_t *
 
     if (l_data_size > p_code_block->data_size) {
         if (p_code_block->data) {
-            opj_free(p_code_block->data - 1); /* again, why -1 */
+            /* We refer to data - 1 since below we incremented it */
+            opj_free(p_code_block->data - 1);
         }
         p_code_block->data = (OPJ_BYTE*) opj_malloc(l_data_size + 1);
         if (! p_code_block->data) {
@@ -1172,6 +1197,10 @@ static OPJ_BOOL opj_tcd_code_block_enc_allocate_data(opj_tcd_cblk_enc_t *
         }
         p_code_block->data_size = l_data_size;
 
+        /* We reserve the initial byte as a fake byte to a non-FF value */
+        /* and increment the data pointer, so that opj_mqc_init_enc() */
+        /* can do bp = data - 1, and opj_mqc_byteout() can safely dereference */
+        /* it. */
         p_code_block->data[0] = 0;
         p_code_block->data += 1; /*why +1 ?*/
     }
@@ -1393,8 +1422,7 @@ OPJ_BOOL opj_tcd_decode_tile(opj_tcd_t *p_tcd,
     /*------------------TIER1-----------------*/
 
     /* FIXME _ProfStart(PGROUP_T1); */
-    if
-    (! opj_tcd_t1_decode(p_tcd)) {
+    if (! opj_tcd_t1_decode(p_tcd, p_manager)) {
         return OPJ_FALSE;
     }
     /* FIXME _ProfStop(PGROUP_T1); */
@@ -1495,14 +1523,18 @@ OPJ_BOOL opj_tcd_update_tile_data(opj_tcd_t *p_tcd,
             if (l_img_comp->sgnd) {
                 for (j = 0; j < l_height; ++j) {
                     for (k = 0; k < l_width; ++k) {
-                        *(l_dest_ptr++) = (OPJ_INT16)(*(l_src_ptr++));
+                        OPJ_INT16 val = (OPJ_INT16)(*(l_src_ptr++));
+                        memcpy(l_dest_ptr, &val, sizeof(val));
+                        l_dest_ptr ++;
                     }
                     l_src_ptr += l_stride;
                 }
             } else {
                 for (j = 0; j < l_height; ++j) {
                     for (k = 0; k < l_width; ++k) {
-                        *(l_dest_ptr++) = (OPJ_INT16)((*(l_src_ptr++)) & 0xffff);
+                        OPJ_INT16 val = (OPJ_INT16)((*(l_src_ptr++)) & 0xffff);
+                        memcpy(l_dest_ptr, &val, sizeof(val));
+                        l_dest_ptr ++;
                     }
                     l_src_ptr += l_stride;
                 }
@@ -1516,10 +1548,9 @@ OPJ_BOOL opj_tcd_update_tile_data(opj_tcd_t *p_tcd,
             OPJ_INT32 * l_src_ptr = l_tilec->data;
 
             for (j = 0; j < l_height; ++j) {
-                for (k = 0; k < l_width; ++k) {
-                    *(l_dest_ptr++) = (*(l_src_ptr++));
-                }
-                l_src_ptr += l_stride;
+                memcpy(l_dest_ptr, l_src_ptr, l_width * sizeof(OPJ_INT32));
+                l_dest_ptr += l_width;
+                l_src_ptr += l_width + l_stride;
             }
 
             p_dest = (OPJ_BYTE*) l_dest_ptr;
@@ -1653,16 +1684,27 @@ static OPJ_BOOL opj_tcd_t2_decode(opj_tcd_t *p_tcd,
     return OPJ_TRUE;
 }
 
-static OPJ_BOOL opj_tcd_t1_decode(opj_tcd_t *p_tcd)
+static OPJ_BOOL opj_tcd_t1_decode(opj_tcd_t *p_tcd, opj_event_mgr_t *p_manager)
 {
     OPJ_UINT32 compno;
     opj_tcd_tile_t * l_tile = p_tcd->tcd_image->tiles;
     opj_tcd_tilecomp_t* l_tile_comp = l_tile->comps;
     opj_tccp_t * l_tccp = p_tcd->tcp->tccps;
     volatile OPJ_BOOL ret = OPJ_TRUE;
+    OPJ_BOOL check_pterm = OPJ_FALSE;
+    opj_mutex_t* p_manager_mutex = NULL;
+
+    p_manager_mutex = opj_mutex_create();
+
+    /* Only enable PTERM check if we decode all layers */
+    if (p_tcd->tcp->num_layers_to_decode == p_tcd->tcp->numlayers &&
+            (l_tccp->cblksty & J2K_CCP_CBLKSTY_PTERM) != 0) {
+        check_pterm = OPJ_TRUE;
+    }
 
     for (compno = 0; compno < l_tile->numcomps; ++compno) {
-        opj_t1_decode_cblks(p_tcd->thread_pool, &ret, l_tile_comp, l_tccp);
+        opj_t1_decode_cblks(p_tcd->thread_pool, &ret, l_tile_comp, l_tccp,
+                            p_manager, p_manager_mutex, check_pterm);
         if (!ret) {
             break;
         }
@@ -1671,7 +1713,9 @@ static OPJ_BOOL opj_tcd_t1_decode(opj_tcd_t *p_tcd)
     }
 
     opj_thread_pool_wait_completion(p_tcd->thread_pool, 0);
-
+    if (p_manager_mutex) {
+        opj_mutex_destroy(p_manager_mutex);
+    }
     return ret;
 }
 
@@ -1846,8 +1890,15 @@ static OPJ_BOOL opj_tcd_dc_level_shift_decode(opj_tcd_t *p_tcd)
             for (j = 0; j < l_height; ++j) {
                 for (i = 0; i < l_width; ++i) {
                     OPJ_FLOAT32 l_value = *((OPJ_FLOAT32 *) l_current_ptr);
-                    *l_current_ptr = opj_int_clamp((OPJ_INT32)opj_lrintf(l_value) +
-                                                   l_tccp->m_dc_level_shift, l_min, l_max); ;
+                    OPJ_INT32 l_value_int = (OPJ_INT32)opj_lrintf(l_value);
+                    if (l_value > INT_MAX ||
+                            (l_value_int > 0 && l_tccp->m_dc_level_shift > 0 &&
+                             l_value_int > INT_MAX - l_tccp->m_dc_level_shift)) {
+                        *l_current_ptr = l_max;
+                    } else {
+                        *l_current_ptr = opj_int_clamp(
+                                             l_value_int + l_tccp->m_dc_level_shift, l_min, l_max);
+                    }
                     ++l_current_ptr;
                 }
                 l_current_ptr += l_stride;
@@ -1915,6 +1966,8 @@ static void opj_tcd_code_block_enc_deallocate(opj_tcd_precinct_t * p_precinct)
 
         for (cblkno = 0; cblkno < l_nb_code_blocks; ++cblkno)  {
             if (l_code_block->data) {
+                /* We refer to data - 1 since below we incremented it */
+                /* in opj_tcd_code_block_enc_allocate_data() */
                 opj_free(l_code_block->data - 1);
                 l_code_block->data = 00;
             }
@@ -2274,3 +2327,8 @@ OPJ_BOOL opj_tcd_copy_tile_data(opj_tcd_t *p_tcd,
 
     return OPJ_TRUE;
 }
+
+OPJ_BOOL opj_tcd_is_band_empty(opj_tcd_band_t* band)
+{
+    return (band->x1 - band->x0 == 0) || (band->y1 - band->y0 == 0);
+}