diff options
| author | Even Rouault <even.rouault@spatialys.com> | 2017-09-01 19:27:56 +0200 |
|---|---|---|
| committer | Even Rouault <even.rouault@spatialys.com> | 2017-09-01 19:27:56 +0200 |
| commit | a538815c774db94a0d3fa323c9ca4f456ef04903 (patch) | |
| tree | fc34a5af855d3b97ec47b24bc8acd218470d6f72 /src | |
| parent | 09929bb615bd6bf2a9f7703fc7b04a12096b091e (diff) | |
opj_j2k_setup_encoder(): avoid potential int overflow in computations related to max_cs_size
Diffstat (limited to 'src')
| -rw-r--r-- | src/lib/openjp2/j2k.c | 22 |
1 files changed, 13 insertions, 9 deletions
diff --git a/src/lib/openjp2/j2k.c b/src/lib/openjp2/j2k.c index 1e9f3695..a551f496 100644 --- a/src/lib/openjp2/j2k.c +++ b/src/lib/openjp2/j2k.c @@ -6812,21 +6812,25 @@ OPJ_BOOL opj_j2k_setup_encoder(opj_j2k_t *p_j2k, if (parameters->max_cs_size <= 0) { if (parameters->tcp_rates[parameters->tcp_numlayers - 1] > 0) { OPJ_FLOAT32 temp_size; - temp_size = (OPJ_FLOAT32)(image->numcomps * image->comps[0].w * - image->comps[0].h * image->comps[0].prec) / - (parameters->tcp_rates[parameters->tcp_numlayers - 1] * 8 * - (OPJ_FLOAT32)image->comps[0].dx * (OPJ_FLOAT32)image->comps[0].dy); - parameters->max_cs_size = (int) floor(temp_size); + temp_size = (OPJ_FLOAT32)(((double)image->numcomps * image->comps[0].w * + image->comps[0].h * image->comps[0].prec) / + ((double)parameters->tcp_rates[parameters->tcp_numlayers - 1] * 8 * + image->comps[0].dx * image->comps[0].dy)); + if (temp_size > INT_MAX) { + parameters->max_cs_size = INT_MAX; + } else { + parameters->max_cs_size = (int) floor(temp_size); + } } else { parameters->max_cs_size = 0; } } else { OPJ_FLOAT32 temp_rate; OPJ_BOOL cap = OPJ_FALSE; - temp_rate = (OPJ_FLOAT32)(image->numcomps * image->comps[0].w * - image->comps[0].h * image->comps[0].prec) / - (OPJ_FLOAT32)(((OPJ_UINT32)parameters->max_cs_size) * 8 * image->comps[0].dx * - image->comps[0].dy); + temp_rate = (OPJ_FLOAT32)(((double)image->numcomps * image->comps[0].w * + image->comps[0].h * image->comps[0].prec) / + (((double)parameters->max_cs_size) * 8 * image->comps[0].dx * + image->comps[0].dy)); for (i = 0; i < (OPJ_UINT32) parameters->tcp_numlayers; i++) { if (parameters->tcp_rates[i] < temp_rate) { parameters->tcp_rates[i] = temp_rate; |