Fix some potential overflow issues (#1161)

* Fix some potential overflow issues

Put sizeof to the beginning of the multiplication to enforce that
size_t instead of smaller integer types is used for the calculation.

This fixes warnings from LGTM:

    Multiplication result may overflow 'unsigned int'
    before it is converted to 'unsigned long'.

It also allows removing some type casts.

Signed-off-by: Stefan Weil <sw@weilnetz.de>

* Fix code indentation

Signed-off-by: Stefan Weil <sw@weilnetz.de>

8 files changed, 22 insertions, 23 deletions

diff --git a/src/bin/jp2/convertbmp.c b/src/bin/jp2/convertbmp.c
index 7fde99ab..85a47fea 100644
--- a/src/bin/jp2/convertbmp.c
+++ b/src/bin/jp2/convertbmp.c
@@ -763,7 +763,7 @@ opj_image_t* bmptoimage(const char *filename, opj_cparameters_t *parameters)
         fclose(IN);
         return NULL;
     }
-    pData = (OPJ_UINT8 *) calloc(1, stride * Info_h.biHeight * sizeof(OPJ_UINT8));
+    pData = (OPJ_UINT8 *) calloc(1, sizeof(OPJ_UINT8) * stride * Info_h.biHeight);
     if (pData == NULL) {
         fclose(IN);
         return NULL;
diff --git a/src/bin/jp2/converttif.c b/src/bin/jp2/converttif.c
index 6e5c4f40..6714d69c 100644
--- a/src/bin/jp2/converttif.c
+++ b/src/bin/jp2/converttif.c
@@ -725,8 +725,7 @@ int imagetotif(opj_image_t * image, const char *outfile)
         TIFFClose(tif);
         return 1;
     }
-    buffer32s = (OPJ_INT32 *)malloc((OPJ_SIZE_T)(width * numcomps * sizeof(
-                                        OPJ_INT32)));
+    buffer32s = (OPJ_INT32 *)malloc(sizeof(OPJ_INT32) * width * numcomps);
     if (buffer32s == NULL) {
         _TIFFfree(buf);
         TIFFClose(tif);
@@ -1447,8 +1446,7 @@ opj_image_t* tiftoimage(const char *filename, opj_cparameters_t *parameters)
     }
 
     rowStride = (int64_t)((tiWidth * tiSpp * tiBps + 7U) / 8U);
-    buffer32s = (OPJ_INT32 *)malloc((OPJ_SIZE_T)(tiWidth * tiSpp * sizeof(
-                                        OPJ_INT32)));
+    buffer32s = (OPJ_INT32 *)malloc(sizeof(OPJ_INT32) * tiWidth * tiSpp);
     if (buffer32s == NULL) {
         _TIFFfree(buf);
         TIFFClose(tif);
diff --git a/src/bin/jp2/opj_compress.c b/src/bin/jp2/opj_compress.c
index 2cc2970f..148bf951 100644
--- a/src/bin/jp2/opj_compress.c
+++ b/src/bin/jp2/opj_compress.c
@@ -824,7 +824,7 @@ static int parse_cmdline_encoder(int argc, char **argv,
             parameters->tcp_numlayers = (int)numlayers;
             numresolution = (OPJ_UINT32)parameters->numresolution;
             matrix_width = numresolution * 3;
-            parameters->cp_matrice = (int *) malloc(numlayers * matrix_width * sizeof(int));
+            parameters->cp_matrice = (int *) malloc(sizeof(int) * numlayers * matrix_width);
             if (parameters->cp_matrice == NULL) {
                 return 1;
             }
diff --git a/src/bin/jp2/opj_decompress.c b/src/bin/jp2/opj_decompress.c
index a28a194b..4b9583b7 100644
--- a/src/bin/jp2/opj_decompress.c
+++ b/src/bin/jp2/opj_decompress.c
@@ -1119,11 +1119,11 @@ static opj_image_t* convert_gray_to_rgb(opj_image_t* original)
             l_new_image->comps[2].resno_decoded = original->comps[0].resno_decoded;
 
     memcpy(l_new_image->comps[0].data, original->comps[0].data,
-           original->comps[0].w * original->comps[0].h * sizeof(OPJ_INT32));
+           sizeof(OPJ_INT32) * original->comps[0].w * original->comps[0].h);
     memcpy(l_new_image->comps[1].data, original->comps[0].data,
-           original->comps[0].w * original->comps[0].h * sizeof(OPJ_INT32));
+           sizeof(OPJ_INT32) * original->comps[0].w * original->comps[0].h);
     memcpy(l_new_image->comps[2].data, original->comps[0].data,
-           original->comps[0].w * original->comps[0].h * sizeof(OPJ_INT32));
+           sizeof(OPJ_INT32) * original->comps[0].w * original->comps[0].h);
 
     for (compno = 1U; compno < original->numcomps; ++compno) {
         l_new_image->comps[compno + 2U].factor        = original->comps[compno].factor;
@@ -1131,7 +1131,7 @@ static opj_image_t* convert_gray_to_rgb(opj_image_t* original)
         l_new_image->comps[compno + 2U].resno_decoded =
             original->comps[compno].resno_decoded;
         memcpy(l_new_image->comps[compno + 2U].data, original->comps[compno].data,
-               original->comps[compno].w * original->comps[compno].h * sizeof(OPJ_INT32));
+               sizeof(OPJ_INT32) * original->comps[compno].w * original->comps[compno].h);
     }
     opj_image_destroy(original);
     return l_new_image;
@@ -1301,7 +1301,7 @@ static opj_image_t* upsample_image_components(opj_image_t* original)
             }
         } else {
             memcpy(l_new_cmp->data, l_org_cmp->data,
-                   l_org_cmp->w * l_org_cmp->h * sizeof(OPJ_INT32));
+                   sizeof(OPJ_INT32) * l_org_cmp->w * l_org_cmp->h);
         }
     }
     opj_image_destroy(original);
@@ -1360,8 +1360,9 @@ int main(int argc, char **argv)
             destroy_parameters(&parameters);
             return EXIT_FAILURE;
         }
-        dirptr->filename_buf = (char*)malloc((size_t)num_images * OPJ_PATH_LEN * sizeof(
-                char)); /* Stores at max 10 image file names*/
+        /* Stores at max 10 image file names */
+        dirptr->filename_buf = (char*)malloc(sizeof(char) *
+                                             (size_t)num_images * OPJ_PATH_LEN);
         if (!dirptr->filename_buf) {
             failed = 1;
             goto fin;
diff --git a/src/lib/openjp2/jp2.c b/src/lib/openjp2/jp2.c
index 180a416b..4402ffe3 100644
--- a/src/lib/openjp2/jp2.c
+++ b/src/lib/openjp2/jp2.c
@@ -1079,7 +1079,7 @@ static OPJ_BOOL opj_jp2_apply_pclr(opj_image_t *image,
 
         /* Palette mapping: */
         new_comps[i].data = (OPJ_INT32*)
-                            opj_image_data_alloc(old_comps[cmp].w * old_comps[cmp].h * sizeof(OPJ_INT32));
+                            opj_image_data_alloc(sizeof(OPJ_INT32) * old_comps[cmp].w * old_comps[cmp].h);
         if (!new_comps[i].data) {
             while (i > 0) {
                 -- i;
@@ -1193,8 +1193,8 @@ static OPJ_BOOL opj_jp2_read_pclr(opj_jp2_t *jp2,
         return OPJ_FALSE;
     }
 
-    entries = (OPJ_UINT32*) opj_malloc((size_t)nr_channels * nr_entries * sizeof(
-                                           OPJ_UINT32));
+    entries = (OPJ_UINT32*) opj_malloc(sizeof(OPJ_UINT32) * nr_channels *
+                                       nr_entries);
     if (!entries) {
         return OPJ_FALSE;
     }
diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index 76744380..ec04c682 100644
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -1618,8 +1618,8 @@ static void opj_t1_clbl_decode_processor(void* user_data, opj_tls_t* tls)
         cblk_w = (OPJ_UINT32)(cblk->x1 - cblk->x0);
         cblk_h = (OPJ_UINT32)(cblk->y1 - cblk->y0);
 
-        cblk->decoded_data = (OPJ_INT32*)opj_aligned_malloc(cblk_w * cblk_h * sizeof(
-                                 OPJ_INT32));
+        cblk->decoded_data = (OPJ_INT32*)opj_aligned_malloc(sizeof(OPJ_INT32) *
+                             cblk_w * cblk_h);
         if (cblk->decoded_data == NULL) {
             if (job->p_manager_mutex) {
                 opj_mutex_lock(job->p_manager_mutex);
@@ -1634,7 +1634,7 @@ static void opj_t1_clbl_decode_processor(void* user_data, opj_tls_t* tls)
             return;
         }
         /* Zero-init required */
-        memset(cblk->decoded_data, 0, cblk_w * cblk_h * sizeof(OPJ_INT32));
+        memset(cblk->decoded_data, 0, sizeof(OPJ_INT32) * cblk_w * cblk_h);
     } else if (cblk->decoded_data) {
         /* Not sure if that code path can happen, but better be */
         /* safe than sorry */
diff --git a/src/lib/openmj2/j2k.c b/src/lib/openmj2/j2k.c
index 85f7b40e..daf13f80 100644
--- a/src/lib/openmj2/j2k.c
+++ b/src/lib/openmj2/j2k.c
@@ -331,7 +331,7 @@ static int j2k_calculate_tp(opj_cp_t *cp, int img_numcomp, opj_image_t *image,
 
     OPJ_ARG_NOT_USED(img_numcomp);
 
-    j2k->cur_totnum_tp = (int *) opj_malloc(cp->tw * cp->th * sizeof(int));
+    j2k->cur_totnum_tp = (int *) opj_malloc(sizeof(int) * cp->tw * cp->th);
     for (tileno = 0; tileno < cp->tw * cp->th; tileno++) {
         int cur_totnum_tp = 0;
         opj_tcp_t *tcp = &cp->tcps[tileno];
@@ -611,7 +611,7 @@ static void j2k_read_siz(opj_j2k_t *j2k)
         opj_event_msg(j2k->cinfo, EVT_ERROR, "Out of memory\n");
         return;
     }
-    cp->tileno = (int*) opj_malloc(cp->tw * cp->th * sizeof(int));
+    cp->tileno = (int*) opj_malloc(sizeof(int) * cp->tw * cp->th);
     if (cp->tileno == NULL) {
         opj_event_msg(j2k->cinfo, EVT_ERROR, "Out of memory\n");
         return;
diff --git a/src/lib/openmj2/jp2.c b/src/lib/openmj2/jp2.c
index d2ca41f0..acf64b9a 100644
--- a/src/lib/openmj2/jp2.c
+++ b/src/lib/openmj2/jp2.c
@@ -393,7 +393,7 @@ static void jp2_apply_pclr(opj_jp2_color_t *color, opj_image_t *image,
         }
         /* Palette mapping: */
         new_comps[pcol].data = (int*)
-                               opj_malloc(old_comps[cmp].w * old_comps[cmp].h * sizeof(int));
+                               opj_malloc(sizeof(int) * old_comps[cmp].w * old_comps[cmp].h);
         new_comps[pcol].prec = channel_size[i];
         new_comps[pcol].sgnd = channel_sign[i];
     }
@@ -461,7 +461,7 @@ static opj_bool jp2_read_pclr(opj_jp2_t *jp2, opj_cio_t *cio,
     nr_channels = (unsigned short)cio_read(cio, 1);/* NPC */
 
     entries = (unsigned int*)
-              opj_malloc(nr_channels * nr_entries * sizeof(unsigned int));
+              opj_malloc(sizeof(unsigned int) * nr_channels * nr_entries);
     channel_size = (unsigned char*)opj_malloc(nr_channels);
     channel_sign = (unsigned char*)opj_malloc(nr_channels);