Avoid heap buffer overflow in function pnmtoimage of convert.c, and unsigned integer overflow in opj_image_create() (CVE-2016-9118, #861)

1 files changed, 7 insertions, 1 deletions

diff --git a/src/lib/openjp2/image.c b/src/lib/openjp2/image.c
index e62b416c..d00a2370 100644
--- a/src/lib/openjp2/image.c
+++ b/src/lib/openjp2/image.c
@@ -68,7 +68,13 @@ opj_image_t* OPJ_CALLCONV opj_image_create(OPJ_UINT32 numcmpts,
             comp->prec = cmptparms[compno].prec;
             comp->bpp = cmptparms[compno].bpp;
             comp->sgnd = cmptparms[compno].sgnd;
-            comp->data = (OPJ_INT32*) opj_calloc(comp->w * comp->h, sizeof(OPJ_INT32));
+            if (comp->h != 0 && (OPJ_SIZE_T)comp->w > SIZE_MAX / comp->h) {
+                // TODO event manager
+                opj_image_destroy(image);
+                return NULL;
+            }
+            comp->data = (OPJ_INT32*) opj_calloc((OPJ_SIZE_T)comp->w * comp->h,
+                                                 sizeof(OPJ_INT32));
             if (!comp->data) {
                 /* TODO replace with event manager, breaks API */
                 /* fprintf(stderr,"Unable to allocate memory for image.\n"); */