Fix warnings from recent overflow checks

see #832

2 files changed, 27 insertions, 24 deletions

diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c
index b39f63e7..b7d5dc2f 100644
--- a/src/lib/openjp2/dwt.c
+++ b/src/lib/openjp2/dwt.c
@@ -395,7 +395,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 
 	OPJ_INT32 rw;			/* width of the resolution level computed   */
 	OPJ_INT32 rh;			/* height of the resolution level computed  */
-	OPJ_UINT32 l_data_size;
+    size_t l_data_size;
 
 	opj_tcd_resolution_t * l_cur_res = 0;
 	opj_tcd_resolution_t * l_last_res = 0;
@@ -410,12 +410,13 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 	l_data_size = opj_dwt_max_resolution(tilec->resolutions, tilec->numresolutions);
  
 	/* overflow check */
-	if ((size_t)l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+    if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
 
-	bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size * sizeof(OPJ_INT32));
+    l_data_size *= sizeof(OPJ_INT32);
+    bj = (OPJ_INT32*)opj_malloc(l_data_size);
 	/* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
 	/* in that case, so do not error out */
 	if (l_data_size != 0 && ! bj) {
@@ -578,21 +579,22 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 numres
 
 	OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
 
-	OPJ_UINT32 mr; /* max resolution */
+    size_t mr; /* max resolution */
 	
 	if (numres == 1U) {
 		return OPJ_TRUE;
 	}
 	
-	mr = opj_dwt_max_resolution(tr, numres);
+    mr = opj_dwt_max_resolution(tr, numres);
 	
 	/* overflow check */
-	if ((size_t)mr > (SIZE_MAX / sizeof(OPJ_INT32))) {
+    if (mr > (SIZE_MAX / sizeof(OPJ_INT32))) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
 	
-	h.mem = (OPJ_INT32*)opj_aligned_malloc((size_t)mr * sizeof(OPJ_INT32));
+    mr *= sizeof(OPJ_INT32);
+    h.mem = (OPJ_INT32*)opj_aligned_malloc(mr);
 	if (! h.mem){
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
@@ -865,7 +867,7 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
 
 	OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
 	
-	OPJ_UINT32 mr; /* max resolution */
+    size_t mr; /* max resolution */
 	
 	mr = opj_dwt_max_resolution(res, numres);
 	
@@ -876,12 +878,13 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
 	}
 	mr += 5U;
 	
-	if ((size_t)mr > (SIZE_MAX / sizeof(opj_v4_t))) {
+    if (mr > (SIZE_MAX / sizeof(opj_v4_t))) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
 
-	h.wavelet = (opj_v4_t*) opj_aligned_malloc((size_t)mr * sizeof(opj_v4_t));
+    mr *= sizeof(opj_v4_t);
+    h.wavelet = (opj_v4_t*) opj_aligned_malloc(mr);
 	if (!h.wavelet) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index 211708fc..bd3acf7e 100644
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -1166,37 +1166,37 @@ static OPJ_BOOL opj_t1_allocate_buffers(
 		OPJ_UINT32 w,
 		OPJ_UINT32 h)
 {
-	OPJ_UINT32 flagssize;
+    size_t flagssize;
 
 	/* encoder uses tile buffer, so no need to allocate */
 	if (!t1->encoder) {
-		OPJ_UINT32 datasize;
+        size_t datasize;
 		
 		/* Overflow check */
 		if ((w > 0U) && (h > (0xFFFFFFFFU /* UINT32_MAX */ / w))) {
 			/* FIXME event manager error callback */
 			return OPJ_FALSE;
 		}
-		datasize = w * h;
+        datasize = (size_t) w * h;
 		
 		/* Overflow check */
-		if ((size_t)datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
+        if (datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
 			/* FIXME event manager error callback */
 			return OPJ_FALSE;
 		}
 		
-		if(datasize > t1->datasize){
+        if(datasize > (size_t) t1->datasize){
 			opj_aligned_free(t1->data);
-			t1->data = (OPJ_INT32*) opj_aligned_malloc((size_t)datasize * sizeof(OPJ_INT32));
+            t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32));
 			if(!t1->data){
 				/* FIXME event manager error callback */
 				return OPJ_FALSE;
 			}
-			t1->datasize = datasize;
+            t1->datasize = (OPJ_UINT32) datasize;
 		}
 		/* memset first arg is declared to never be null by gcc */
 		if (t1->data != NULL) {
-			memset(t1->data, 0, (size_t)datasize * sizeof(OPJ_INT32));
+            memset(t1->data, 0, datasize * sizeof(OPJ_INT32));
 		}
 	}
 	
@@ -1210,24 +1210,24 @@ static OPJ_BOOL opj_t1_allocate_buffers(
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
-	flagssize = t1->flags_stride * (h + 2U);
+    flagssize = (size_t) t1->flags_stride * (h + 2U);
 
 	/* Overflow check */
-	if ((size_t)flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
+    if (flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
 		/* FIXME event manager error callback */
 		return OPJ_FALSE;
 	}
 	
-	if(flagssize > t1->flagssize){
+    if(flagssize > (size_t) t1->flagssize){
 		opj_aligned_free(t1->flags);
-		t1->flags = (opj_flag_t*) opj_aligned_malloc((size_t)flagssize * sizeof(opj_flag_t));
+        t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t));
 		if(!t1->flags){
 			/* FIXME event manager error callback */
 			return OPJ_FALSE;
 		}
-		t1->flagssize = flagssize;
+        t1->flagssize = (OPJ_UINT32) flagssize;
 	}
-	memset(t1->flags, 0, (size_t)flagssize * sizeof(opj_flag_t));
+    memset(t1->flags, 0, flagssize * sizeof(opj_flag_t));
 
 	t1->w = w;
 	t1->h = h;