summaryrefslogtreecommitdiff
path: root/src/lib/openjp2/tcd.c
diff options
context:
space:
mode:
authorEven Rouault <even.rouault@spatialys.com>2017-08-28 17:18:33 +0200
committerEven Rouault <even.rouault@spatialys.com>2017-08-28 17:18:33 +0200
commita55c024fc6b917a409b85aeafd7326421c4aea34 (patch)
treee365ae7953a37341e0413d992a3b3df7cf2627ed /src/lib/openjp2/tcd.c
parent8f92fc97913bec7ffa2dc10d062c0cdd19da20e4 (diff)
Subtile decoding: fix overflows in subband coordinate computation that cause later buffer overflow. Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=3115. Credit to OSS Fuzz. master only
Diffstat (limited to 'src/lib/openjp2/tcd.c')
-rw-r--r--src/lib/openjp2/tcd.c20
1 files changed, 12 insertions, 8 deletions
diff --git a/src/lib/openjp2/tcd.c b/src/lib/openjp2/tcd.c
index 7a5e2b44..1c56c1b3 100644
--- a/src/lib/openjp2/tcd.c
+++ b/src/lib/openjp2/tcd.c
@@ -2414,14 +2414,18 @@ OPJ_BOOL opj_tcd_is_subband_area_of_interest(opj_tcd_t *tcd,
/* equation B-15 of the standard */
OPJ_UINT32 x0b = bandno & 1;
OPJ_UINT32 y0b = bandno >> 1;
- OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 : opj_uint_ceildiv(tcx0 - (1U <<
- (nb - 1)) * x0b, 1U << nb);
- OPJ_UINT32 tby0 = (nb == 0) ? tcy0 : opj_uint_ceildiv(tcy0 - (1U <<
- (nb - 1)) * y0b, 1U << nb);
- OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 : opj_uint_ceildiv(tcx1 - (1U <<
- (nb - 1)) * x0b, 1U << nb);
- OPJ_UINT32 tby1 = (nb == 0) ? tcy1 : opj_uint_ceildiv(tcy1 - (1U <<
- (nb - 1)) * y0b, 1U << nb);
+ OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 :
+ (tcx0 <= (1U << (nb - 1)) * x0b) ? 0 :
+ opj_uint_ceildivpow2(tcx0 - (1U << (nb - 1)) * x0b, nb);
+ OPJ_UINT32 tby0 = (nb == 0) ? tcy0 :
+ (tcy0 <= (1U << (nb - 1)) * y0b) ? 0 :
+ opj_uint_ceildivpow2(tcy0 - (1U << (nb - 1)) * y0b, nb);
+ OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 :
+ (tcx1 <= (1U << (nb - 1)) * x0b) ? 0 :
+ opj_uint_ceildivpow2(tcx1 - (1U << (nb - 1)) * x0b, nb);
+ OPJ_UINT32 tby1 = (nb == 0) ? tcy1 :
+ (tcy1 <= (1U << (nb - 1)) * y0b) ? 0 :
+ opj_uint_ceildivpow2(tcy1 - (1U << (nb - 1)) * y0b, nb);
OPJ_BOOL intersects;
if (tbx0 < filter_margin) {