diff options
| author | Even Rouault <even.rouault@spatialys.com> | 2017-08-28 17:18:33 +0200 |
|---|---|---|
| committer | Even Rouault <even.rouault@spatialys.com> | 2017-08-28 17:18:33 +0200 |
| commit | a55c024fc6b917a409b85aeafd7326421c4aea34 (patch) | |
| tree | e365ae7953a37341e0413d992a3b3df7cf2627ed /src | |
| parent | 8f92fc97913bec7ffa2dc10d062c0cdd19da20e4 (diff) | |
Subtile decoding: fix overflows in subband coordinate computation that cause later buffer overflow. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=3115. Credit to OSS Fuzz. master only
Diffstat (limited to 'src')
| -rw-r--r-- | src/lib/openjp2/dwt.c | 28 | ||||
| -rw-r--r-- | src/lib/openjp2/tcd.c | 20 |
2 files changed, 27 insertions, 21 deletions
diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c index d33fbeb3..de3ff727 100644 --- a/src/lib/openjp2/dwt.c +++ b/src/lib/openjp2/dwt.c @@ -1642,20 +1642,24 @@ static void opj_dwt_get_band_coordinates(opj_tcd_tilecomp_t* tilec, OPJ_UINT32 x0b = bandno & 1; OPJ_UINT32 y0b = bandno >> 1; if (tbx0) { - *tbx0 = (nb == 0) ? tcx0 : opj_uint_ceildiv(tcx0 - (1U << - (nb - 1)) * x0b, 1U << nb); + *tbx0 = (nb == 0) ? tcx0 : + (tcx0 <= (1U << (nb - 1)) * x0b) ? 0 : + opj_uint_ceildivpow2(tcx0 - (1U << (nb - 1)) * x0b, nb); } if (tby0) { - *tby0 = (nb == 0) ? tcy0 : opj_uint_ceildiv(tcy0 - (1U << - (nb - 1)) * y0b, 1U << nb); + *tby0 = (nb == 0) ? tcy0 : + (tcy0 <= (1U << (nb - 1)) * y0b) ? 0 : + opj_uint_ceildivpow2(tcy0 - (1U << (nb - 1)) * y0b, nb); } if (tbx1) { - *tbx1 = (nb == 0) ? tcx1 : opj_uint_ceildiv(tcx1 - (1U << - (nb - 1)) * x0b, 1U << nb); + *tbx1 = (nb == 0) ? tcx1 : + (tcx1 <= (1U << (nb - 1)) * x0b) ? 0 : + opj_uint_ceildivpow2(tcx1 - (1U << (nb - 1)) * x0b, nb); } if (tby1) { - *tby1 = (nb == 0) ? tcy1 : opj_uint_ceildiv(tcy1 - (1U << - (nb - 1)) * y0b, 1U << nb); + *tby1 = (nb == 0) ? tcy1 : + (tcy1 <= (1U << (nb - 1)) * y0b) ? 0 : + opj_uint_ceildivpow2(tcy1 - (1U << (nb - 1)) * y0b, nb); } } @@ -1711,14 +1715,12 @@ static OPJ_BOOL opj_dwt_decode_partial_tile(opj_tcd_t *tcd, } h_mem_size = opj_dwt_max_resolution(tr, numres); /* overflow check */ - if (h_mem_size > (SIZE_MAX / PARALLEL_COLS_53 / sizeof(OPJ_INT32))) { + if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) { /* FIXME event manager error callback */ return OPJ_FALSE; } - /* We need PARALLEL_COLS_53 times the height of the array, */ - /* since for the vertical pass */ - /* we process PARALLEL_COLS_53 columns at a time */ - h_mem_size *= PARALLEL_COLS_53 * sizeof(OPJ_INT32); + + h_mem_size *= sizeof(OPJ_INT32); h.mem = (OPJ_INT32*)opj_aligned_32_malloc(h_mem_size); if (! h.mem) { /* FIXME event manager error callback */ diff --git a/src/lib/openjp2/tcd.c b/src/lib/openjp2/tcd.c index 7a5e2b44..1c56c1b3 100644 --- a/src/lib/openjp2/tcd.c +++ b/src/lib/openjp2/tcd.c @@ -2414,14 +2414,18 @@ OPJ_BOOL opj_tcd_is_subband_area_of_interest(opj_tcd_t *tcd, /* equation B-15 of the standard */ OPJ_UINT32 x0b = bandno & 1; OPJ_UINT32 y0b = bandno >> 1; - OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 : opj_uint_ceildiv(tcx0 - (1U << - (nb - 1)) * x0b, 1U << nb); - OPJ_UINT32 tby0 = (nb == 0) ? tcy0 : opj_uint_ceildiv(tcy0 - (1U << - (nb - 1)) * y0b, 1U << nb); - OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 : opj_uint_ceildiv(tcx1 - (1U << - (nb - 1)) * x0b, 1U << nb); - OPJ_UINT32 tby1 = (nb == 0) ? tcy1 : opj_uint_ceildiv(tcy1 - (1U << - (nb - 1)) * y0b, 1U << nb); + OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 : + (tcx0 <= (1U << (nb - 1)) * x0b) ? 0 : + opj_uint_ceildivpow2(tcx0 - (1U << (nb - 1)) * x0b, nb); + OPJ_UINT32 tby0 = (nb == 0) ? tcy0 : + (tcy0 <= (1U << (nb - 1)) * y0b) ? 0 : + opj_uint_ceildivpow2(tcy0 - (1U << (nb - 1)) * y0b, nb); + OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 : + (tcx1 <= (1U << (nb - 1)) * x0b) ? 0 : + opj_uint_ceildivpow2(tcx1 - (1U << (nb - 1)) * x0b, nb); + OPJ_UINT32 tby1 = (nb == 0) ? tcy1 : + (tcy1 <= (1U << (nb - 1)) * y0b) ? 0 : + opj_uint_ceildivpow2(tcy1 - (1U << (nb - 1)) * y0b, nb); OPJ_BOOL intersects; if (tbx0 < filter_margin) { |