Add overflow check in opj_tcd_init_tile (#819)
[openjpeg.git] / src / lib / openjp2 / tcd.c
index d227e388f1e82259ba65c18606b226ceb0a257bb..12da05cadf32dea541af1659963683f62d34cfc9 100644 (file)
@@ -268,7 +268,7 @@ void opj_tcd_makelayer( opj_tcd_t *tcd,
                                                                         n = passno + 1;
                                                                 continue;
                                                         }
-                                                        if (dd / dr > thresh)
+                                                        if (thresh - (dd / dr) < DBL_EPSILON) /* do not rely on float equality, check with DBL_EPSILON margin */
                                                                 n = passno + 1;
                                                 }
 
@@ -486,8 +486,7 @@ OPJ_BOOL opj_tcd_rateallocate(  opj_tcd_t *tcd,
         for (layno = 0; layno < tcd_tcp->numlayers; layno++) {
                 OPJ_FLOAT64 lo = min;
                 OPJ_FLOAT64 hi = max;
-                OPJ_BOOL success = OPJ_FALSE;
-                OPJ_UINT32 maxlen = tcd_tcp->rates[layno] ? opj_uint_min(((OPJ_UINT32) ceil(tcd_tcp->rates[layno])), len) : len;
+                OPJ_UINT32 maxlen = tcd_tcp->rates[layno] > 0.0f ? opj_uint_min(((OPJ_UINT32) ceil(tcd_tcp->rates[layno])), len) : len;
                 OPJ_FLOAT64 goodthresh = 0;
                 OPJ_FLOAT64 stable_thresh = 0;
                 OPJ_UINT32 i;
@@ -500,7 +499,7 @@ OPJ_BOOL opj_tcd_rateallocate(  opj_tcd_t *tcd,
                   -r xx,yy,zz,0   (disto_alloc == 1 and rates == 0)
                   -q xx,yy,zz,0   (fixed_quality == 1 and distoratio == 0)
                   ==> possible to have some lossy layers and the last layer for sure lossless */
-                if ( ((cp->m_specific_param.m_enc.m_disto_alloc==1) && (tcd_tcp->rates[layno]>0)) || ((cp->m_specific_param.m_enc.m_fixed_quality==1) && (tcd_tcp->distoratio[layno]>0))) {
+                if ( ((cp->m_specific_param.m_enc.m_disto_alloc==1) && (tcd_tcp->rates[layno]>0.0f)) || ((cp->m_specific_param.m_enc.m_fixed_quality==1) && (tcd_tcp->distoratio[layno]>0.0))) {
                         opj_t2_t*t2 = opj_t2_create(tcd->image, cp);
                         OPJ_FLOAT64 thresh = 0;
 
@@ -559,17 +558,11 @@ OPJ_BOOL opj_tcd_rateallocate(  opj_tcd_t *tcd,
                                 }
                         }
 
-                        success = OPJ_TRUE;
                         goodthresh = stable_thresh == 0? thresh : stable_thresh;
 
                         opj_t2_destroy(t2);
                 } else {
-                        success = OPJ_TRUE;
-                        goodthresh = min - DBL_EPSILON; /* do not rely on float equality for lossless transform */
-                }
-
-                if (!success) {
-                        return OPJ_FALSE;
+                        goodthresh = min;
                 }
 
                 if(cstr_info) { /* Threshold for Marcela Index */
@@ -626,7 +619,7 @@ void opj_tcd_destroy(opj_tcd_t *tcd) {
 OPJ_BOOL opj_alloc_tile_component_data(opj_tcd_tilecomp_t *l_tilec)
 {
        if ((l_tilec->data == 00) || ((l_tilec->data_size_needed > l_tilec->data_size) && (l_tilec->ownsData == OPJ_FALSE))) {
-               l_tilec->data = (OPJ_INT32 *) opj_malloc(l_tilec->data_size_needed);
+               l_tilec->data = (OPJ_INT32 *) opj_aligned_malloc(l_tilec->data_size_needed);
                if (! l_tilec->data ) {
                        return OPJ_FALSE;
                }
@@ -635,18 +628,15 @@ OPJ_BOOL opj_alloc_tile_component_data(opj_tcd_tilecomp_t *l_tilec)
                l_tilec->ownsData = OPJ_TRUE;
        }
        else if (l_tilec->data_size_needed > l_tilec->data_size) {
-               OPJ_INT32 * new_data = (OPJ_INT32 *) opj_realloc(l_tilec->data, l_tilec->data_size_needed);
-               /* opj_event_msg(p_manager, EVT_ERROR, "Not enough memory to handle tile datan"); */
-               /* fprintf(stderr, "Not enough memory to handle tile data"); */
-               if (! new_data) {
-                       opj_free(l_tilec->data);
-                       l_tilec->data = NULL;
+               /* We don't need to keep old data */
+               opj_aligned_free(l_tilec->data);
+               l_tilec->data = (OPJ_INT32 *) opj_aligned_malloc(l_tilec->data_size_needed);
+               if (! l_tilec->data ) {
                        l_tilec->data_size = 0;
                        l_tilec->data_size_needed = 0;
                        l_tilec->ownsData = OPJ_FALSE;
                        return OPJ_FALSE;
                }
-               l_tilec->data = new_data;
                /*fprintf(stderr, "tReallocate data of tilec (int): from %d to %d x OPJ_UINT32n", l_tilec->data_size, l_data_size);*/
                l_tilec->data_size = l_tilec->data_size_needed;
                l_tilec->ownsData = OPJ_TRUE;
@@ -730,7 +720,8 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                
                /* compute l_data_size with overflow check */
                l_data_size = (OPJ_UINT32)(l_tilec->x1 - l_tilec->x0);
-               if ((((OPJ_UINT32)-1) / l_data_size) < (OPJ_UINT32)(l_tilec->y1 - l_tilec->y0)) {
+               /* issue 733, l_data_size == 0U, probably something wrong should be checked before getting here */
+               if ((l_data_size > 0U) && ((((OPJ_UINT32)-1) / l_data_size) < (OPJ_UINT32)(l_tilec->y1 - l_tilec->y0))) {
                        opj_event_msg(manager, EVT_ERROR, "Not enough memory for tile data\n");
                        return OPJ_FALSE;
                }
@@ -781,7 +772,7 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                        l_tilec->resolutions_size = l_data_size;
                }
                
-               l_level_no = l_tilec->numresolutions - 1;
+               l_level_no = l_tilec->numresolutions;
                l_res = l_tilec->resolutions;
                l_step_size = l_tccp->stepsizes;
                if (l_tccp->qmfbid == 0) {
@@ -798,6 +789,8 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                        OPJ_UINT32 cbgwidthexpn, cbgheightexpn;
                        OPJ_UINT32 cblkwidthexpn, cblkheightexpn;
                        
+                       --l_level_no;
+                       
                        /* border for each resolution level (global) */
                        l_res->x0 = opj_int_ceildivpow2(l_tilec->x0, (OPJ_INT32)l_level_no);
                        l_res->y0 = opj_int_ceildivpow2(l_tilec->y0, (OPJ_INT32)l_level_no);
@@ -815,12 +808,22 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                        l_br_prc_y_end = opj_int_ceildivpow2(l_res->y1, (OPJ_INT32)l_pdy) << l_pdy;
                        /*fprintf(stderr, "\t\t\tprc_x_start=%d, prc_y_start=%d, br_prc_x_end=%d, br_prc_y_end=%d \n", l_tl_prc_x_start, l_tl_prc_y_start, l_br_prc_x_end ,l_br_prc_y_end );*/
                        
-                       l_res->pw = (l_res->x0 == l_res->x1) ? 0 : (OPJ_UINT32)((l_br_prc_x_end - l_tl_prc_x_start) >> l_pdx);
-                       l_res->ph = (l_res->y0 == l_res->y1) ? 0 : (OPJ_UINT32)((l_br_prc_y_end - l_tl_prc_y_start) >> l_pdy);
+                       l_res->pw = (l_res->x0 == l_res->x1) ? 0U : (OPJ_UINT32)((l_br_prc_x_end - l_tl_prc_x_start) >> l_pdx);
+                       l_res->ph = (l_res->y0 == l_res->y1) ? 0U : (OPJ_UINT32)((l_br_prc_y_end - l_tl_prc_y_start) >> l_pdy);
                        /*fprintf(stderr, "\t\t\tres_pw=%d, res_ph=%d\n", l_res->pw, l_res->ph );*/
-                       
+
+                       if ((l_res->pw != 0U) && ((((OPJ_UINT32)-1) / l_res->pw) < l_res->ph)) {
+                               opj_event_msg(manager, EVT_ERROR, "Not enough memory for tile data\n");
+                               return OPJ_FALSE;
+                       }
                        l_nb_precincts = l_res->pw * l_res->ph;
+
+                       if ((((OPJ_UINT32)-1) / (OPJ_UINT32)sizeof(opj_tcd_precinct_t)) < l_nb_precincts) {
+                               opj_event_msg(manager, EVT_ERROR, "Not enough memory for tile data\n");
+                               return OPJ_FALSE;
+                       }
                        l_nb_precinct_size = l_nb_precincts * (OPJ_UINT32)sizeof(opj_tcd_precinct_t);
+
                        if (resno == 0) {
                                tlcbgxstart = l_tl_prc_x_start;
                                tlcbgystart = l_tl_prc_y_start;
@@ -874,9 +877,10 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                                l_band->stepsize = (OPJ_FLOAT32)(((1.0 + l_step_size->mant / 2048.0) * pow(2.0, (OPJ_INT32) (numbps - l_step_size->expn)))) * fraction;
                                l_band->numbps = l_step_size->expn + (OPJ_INT32)l_tccp->numgbits - 1;      /* WHY -1 ? */
                                
-                               if (! l_band->precincts) {
+                               if (!l_band->precincts && (l_nb_precincts > 0U)) {
                                        l_band->precincts = (opj_tcd_precinct_t *) opj_malloc( /*3 * */ l_nb_precinct_size);
                                        if (! l_band->precincts) {
+                                               opj_event_msg(manager, EVT_ERROR, "Not enough memory to handle band precints\n");
                                                return OPJ_FALSE;
                                        }
                                        /*fprintf(stderr, "\t\t\t\tAllocate precincts of a band (opj_tcd_precinct_t): %d\n",l_nb_precinct_size);     */
@@ -933,7 +937,7 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                                        /*fprintf(stderr, "\t\t\t\t precinct_cw = %d x recinct_ch = %d\n",l_current_precinct->cw, l_current_precinct->ch);      */
                                        l_nb_code_blocks_size = l_nb_code_blocks * (OPJ_UINT32)sizeof_block;
                                        
-                                       if (! l_current_precinct->cblks.blocks) {
+                                       if (!l_current_precinct->cblks.blocks && (l_nb_code_blocks > 0U)) {
                                                l_current_precinct->cblks.blocks = opj_malloc(l_nb_code_blocks_size);
                                                if (! l_current_precinct->cblks.blocks ) {
                                                        return OPJ_FALSE;
@@ -1027,7 +1031,6 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
                                ++l_step_size;
                        } /* bandno */
                        ++l_res;
-                       --l_level_no;
                } /* resno */
                ++l_tccp;
                ++l_tilec;
@@ -1521,7 +1524,7 @@ static void opj_tcd_free_tile(opj_tcd_t *p_tcd)
                 }
 
                 if (l_tile_comp->ownsData && l_tile_comp->data) {
-                        opj_free(l_tile_comp->data);
+                        opj_aligned_free(l_tile_comp->data);
                         l_tile_comp->data = 00;
                         l_tile_comp->ownsData = 0;
                         l_tile_comp->data_size = 0;
@@ -1914,7 +1917,7 @@ static OPJ_BOOL opj_tcd_dc_level_shift_encode ( opj_tcd_t *p_tcd )
                 }
                 else {
                         for (i = 0; i < l_nb_elem; ++i) {
-                                *l_current_ptr = (*l_current_ptr - l_tccp->m_dc_level_shift) << 11 ;
+                                *l_current_ptr = (*l_current_ptr - l_tccp->m_dc_level_shift) * (1 << 11);
                                 ++l_current_ptr;
                         }
                 }