2 Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /** @file src/image.cc
21 * @brief A class to describe a video image.
26 #include <libswscale/swscale.h>
27 #include <libavutil/pixfmt.h>
28 #include <libavutil/pixdesc.h>
31 #include "exceptions.h"
35 #include "md5_digester.h"
44 using boost::shared_ptr;
48 Image::line_factor (int n) const
54 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
56 throw PixelFormatError ("lines()", _pixel_format);
59 return pow (2.0f, d->log2_chroma_h);
62 /** @param n Component index.
63 * @return Number of lines in the image for the given component.
66 Image::lines (int n) const
68 return rint (ceil (static_cast<double>(size().height) / line_factor (n)));
71 /** @return Number of components */
73 Image::components () const
75 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
77 throw PixelFormatError ("components()", _pixel_format);
80 if ((d->flags & PIX_FMT_PLANAR) == 0) {
84 return d->nb_components;
87 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size' */
89 Image::crop_scale_window (Crop crop, dcp::Size inter_size, dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
91 DCPOMATIC_ASSERT (scaler);
92 /* Empirical testing suggests that sws_scale() will crash if
93 the input image is not aligned.
95 DCPOMATIC_ASSERT (aligned ());
97 DCPOMATIC_ASSERT (out_size.width >= inter_size.width);
98 DCPOMATIC_ASSERT (out_size.height >= inter_size.height);
100 /* Here's an image of out_size */
101 shared_ptr<Image> out (new Image (out_format, out_size, out_aligned));
104 /* Size of the image after any crop */
105 dcp::Size const cropped_size = crop.apply (size ());
107 /* Scale context for a scale from cropped_size to inter_size */
108 struct SwsContext* scale_context = sws_getContext (
109 cropped_size.width, cropped_size.height, pixel_format(),
110 inter_size.width, inter_size.height, out_format,
111 scaler->ffmpeg_id (), 0, 0, 0
114 if (!scale_context) {
115 throw StringError (N_("Could not allocate SwsContext"));
118 /* Prepare input data pointers with crop */
119 uint8_t* scale_in_data[components()];
120 for (int c = 0; c < components(); ++c) {
121 scale_in_data[c] = data()[c] + int (rint (bytes_per_pixel(c) * crop.left)) + stride()[c] * (crop.top / line_factor(c));
124 /* Corner of the image within out_size */
125 Position<int> const corner ((out_size.width - inter_size.width) / 2, (out_size.height - inter_size.height) / 2);
127 uint8_t* scale_out_data[out->components()];
128 for (int c = 0; c < out->components(); ++c) {
129 scale_out_data[c] = out->data()[c] + int (rint (out->bytes_per_pixel(c) * corner.x)) + out->stride()[c] * corner.y;
134 scale_in_data, stride(),
135 0, cropped_size.height,
136 scale_out_data, out->stride()
139 sws_freeContext (scale_context);
145 Image::scale (dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
147 DCPOMATIC_ASSERT (scaler);
148 /* Empirical testing suggests that sws_scale() will crash if
149 the input image is not aligned.
151 DCPOMATIC_ASSERT (aligned ());
153 shared_ptr<Image> scaled (new Image (out_format, out_size, out_aligned));
155 struct SwsContext* scale_context = sws_getContext (
156 size().width, size().height, pixel_format(),
157 out_size.width, out_size.height, out_format,
158 scaler->ffmpeg_id (), 0, 0, 0
165 scaled->data(), scaled->stride()
168 sws_freeContext (scale_context);
174 Image::crop (Crop crop, bool aligned) const
176 dcp::Size cropped_size = crop.apply (size ());
177 shared_ptr<Image> out (new Image (pixel_format(), cropped_size, aligned));
179 for (int c = 0; c < components(); ++c) {
180 int const crop_left_in_bytes = bytes_per_pixel(c) * crop.left;
181 /* bytes_per_pixel() could be a fraction; in this case the stride will be rounded
182 up, and we need to make sure that we copy over the width (up to the stride)
183 rather than short of the width; hence the ceil() here.
185 int const cropped_width_in_bytes = ceil (bytes_per_pixel(c) * cropped_size.width);
187 /* Start of the source line, cropped from the top but not the left */
188 uint8_t* in_p = data()[c] + (crop.top / out->line_factor(c)) * stride()[c];
189 uint8_t* out_p = out->data()[c];
191 for (int y = 0; y < out->lines(c); ++y) {
192 memcpy (out_p, in_p + crop_left_in_bytes, cropped_width_in_bytes);
194 out_p += out->stride()[c];
201 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
203 Image::yuv_16_black (uint16_t v, bool alpha)
205 memset (data()[0], 0, lines(0) * stride()[0]);
206 for (int i = 1; i < 3; ++i) {
207 int16_t* p = reinterpret_cast<int16_t*> (data()[i]);
208 for (int y = 0; y < lines(i); ++y) {
209 /* We divide by 2 here because we are writing 2 bytes at a time */
210 for (int x = 0; x < line_size()[i] / 2; ++x) {
213 p += stride()[i] / 2;
218 memset (data()[3], 0, lines(3) * stride()[3]);
223 Image::swap_16 (uint16_t v)
225 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
231 /* U/V black value for 8-bit colour */
232 static uint8_t const eight_bit_uv = (1 << 7) - 1;
233 /* U/V black value for 9-bit colour */
234 static uint16_t const nine_bit_uv = (1 << 8) - 1;
235 /* U/V black value for 10-bit colour */
236 static uint16_t const ten_bit_uv = (1 << 9) - 1;
237 /* U/V black value for 16-bit colour */
238 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
240 switch (_pixel_format) {
241 case PIX_FMT_YUV420P:
242 case PIX_FMT_YUV422P:
243 case PIX_FMT_YUV444P:
244 case PIX_FMT_YUV411P:
245 memset (data()[0], 0, lines(0) * stride()[0]);
246 memset (data()[1], eight_bit_uv, lines(1) * stride()[1]);
247 memset (data()[2], eight_bit_uv, lines(2) * stride()[2]);
250 case PIX_FMT_YUVJ420P:
251 case PIX_FMT_YUVJ422P:
252 case PIX_FMT_YUVJ444P:
253 memset (data()[0], 0, lines(0) * stride()[0]);
254 memset (data()[1], eight_bit_uv + 1, lines(1) * stride()[1]);
255 memset (data()[2], eight_bit_uv + 1, lines(2) * stride()[2]);
258 case PIX_FMT_YUV422P9LE:
259 case PIX_FMT_YUV444P9LE:
260 yuv_16_black (nine_bit_uv, false);
263 case PIX_FMT_YUV422P9BE:
264 case PIX_FMT_YUV444P9BE:
265 yuv_16_black (swap_16 (nine_bit_uv), false);
268 case PIX_FMT_YUV422P10LE:
269 case PIX_FMT_YUV444P10LE:
270 yuv_16_black (ten_bit_uv, false);
273 case PIX_FMT_YUV422P16LE:
274 case PIX_FMT_YUV444P16LE:
275 yuv_16_black (sixteen_bit_uv, false);
278 case PIX_FMT_YUV444P10BE:
279 case PIX_FMT_YUV422P10BE:
280 yuv_16_black (swap_16 (ten_bit_uv), false);
283 case AV_PIX_FMT_YUVA420P9BE:
284 case AV_PIX_FMT_YUVA422P9BE:
285 case AV_PIX_FMT_YUVA444P9BE:
286 yuv_16_black (swap_16 (nine_bit_uv), true);
289 case AV_PIX_FMT_YUVA420P9LE:
290 case AV_PIX_FMT_YUVA422P9LE:
291 case AV_PIX_FMT_YUVA444P9LE:
292 yuv_16_black (nine_bit_uv, true);
295 case AV_PIX_FMT_YUVA420P10BE:
296 case AV_PIX_FMT_YUVA422P10BE:
297 case AV_PIX_FMT_YUVA444P10BE:
298 yuv_16_black (swap_16 (ten_bit_uv), true);
301 case AV_PIX_FMT_YUVA420P10LE:
302 case AV_PIX_FMT_YUVA422P10LE:
303 case AV_PIX_FMT_YUVA444P10LE:
304 yuv_16_black (ten_bit_uv, true);
307 case AV_PIX_FMT_YUVA420P16BE:
308 case AV_PIX_FMT_YUVA422P16BE:
309 case AV_PIX_FMT_YUVA444P16BE:
310 yuv_16_black (swap_16 (sixteen_bit_uv), true);
313 case AV_PIX_FMT_YUVA420P16LE:
314 case AV_PIX_FMT_YUVA422P16LE:
315 case AV_PIX_FMT_YUVA444P16LE:
316 yuv_16_black (sixteen_bit_uv, true);
324 case PIX_FMT_RGB555LE:
325 case PIX_FMT_RGB48LE:
326 case PIX_FMT_RGB48BE:
327 memset (data()[0], 0, lines(0) * stride()[0]);
330 case PIX_FMT_UYVY422:
332 int const Y = lines(0);
333 int const X = line_size()[0];
334 uint8_t* p = data()[0];
335 for (int y = 0; y < Y; ++y) {
336 for (int x = 0; x < X / 4; ++x) {
337 *p++ = eight_bit_uv; // Cb
339 *p++ = eight_bit_uv; // Cr
347 throw PixelFormatError ("make_black()", _pixel_format);
352 Image::make_transparent ()
354 if (_pixel_format != PIX_FMT_RGBA) {
355 throw PixelFormatError ("make_transparent()", _pixel_format);
358 memset (data()[0], 0, lines(0) * stride()[0]);
362 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
364 DCPOMATIC_ASSERT (other->pixel_format() == PIX_FMT_RGBA);
365 int const other_bpp = 4;
367 int start_tx = position.x;
371 start_ox = -start_tx;
375 int start_ty = position.y;
379 start_oy = -start_ty;
383 switch (_pixel_format) {
386 int const this_bpp = 3;
387 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
388 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
389 uint8_t* op = other->data()[0] + oy * other->stride()[0];
390 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
391 float const alpha = float (op[3]) / 255;
392 tp[0] = op[0] + (tp[0] * (1 - alpha));
393 tp[1] = op[1] + (tp[1] * (1 - alpha));
394 tp[2] = op[2] + (tp[2] * (1 - alpha));
405 int const this_bpp = 4;
406 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
407 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
408 uint8_t* op = other->data()[0] + oy * other->stride()[0];
409 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
410 float const alpha = float (op[3]) / 255;
411 tp[0] = op[0] + (tp[0] * (1 - alpha));
412 tp[1] = op[1] + (tp[1] * (1 - alpha));
413 tp[2] = op[2] + (tp[2] * (1 - alpha));
414 tp[3] = op[3] + (tp[3] * (1 - alpha));
422 case PIX_FMT_RGB48LE:
424 int const this_bpp = 6;
425 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
426 uint8_t* tp = data()[0] + ty * stride()[0] + start_tx * this_bpp;
427 uint8_t* op = other->data()[0] + oy * other->stride()[0];
428 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
429 float const alpha = float (op[3]) / 255;
430 /* Blend high bytes */
431 tp[1] = op[0] + (tp[1] * (1 - alpha));
432 tp[3] = op[1] + (tp[3] * (1 - alpha));
433 tp[5] = op[2] + (tp[5] * (1 - alpha));
442 DCPOMATIC_ASSERT (false);
447 Image::copy (shared_ptr<const Image> other, Position<int> position)
449 /* Only implemented for RGB24 onto RGB24 so far */
450 DCPOMATIC_ASSERT (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGB24);
451 DCPOMATIC_ASSERT (position.x >= 0 && position.y >= 0);
453 int const N = min (position.x + other->size().width, size().width) - position.x;
454 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
455 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
456 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
457 memcpy (tp, op, N * 3);
462 Image::read_from_socket (shared_ptr<Socket> socket)
464 for (int i = 0; i < components(); ++i) {
465 uint8_t* p = data()[i];
466 for (int y = 0; y < lines(i); ++y) {
467 socket->read (p, line_size()[i]);
474 Image::write_to_socket (shared_ptr<Socket> socket) const
476 for (int i = 0; i < components(); ++i) {
477 uint8_t* p = data()[i];
478 for (int y = 0; y < lines(i); ++y) {
479 socket->write (p, line_size()[i]);
487 Image::bytes_per_pixel (int c) const
489 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
491 throw PixelFormatError ("lines()", _pixel_format);
494 if (c >= components()) {
498 float bpp[4] = { 0, 0, 0, 0 };
500 bpp[0] = floor ((d->comp[0].depth_minus1 + 1 + 7) / 8);
501 if (d->nb_components > 1) {
502 bpp[1] = floor ((d->comp[1].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
504 if (d->nb_components > 2) {
505 bpp[2] = floor ((d->comp[2].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
507 if (d->nb_components > 3) {
508 bpp[3] = floor ((d->comp[3].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
511 if ((d->flags & PIX_FMT_PLANAR) == 0) {
512 /* Not planar; sum them up */
513 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
519 /** Construct a Image of a given size and format, allocating memory
522 * @param p Pixel format.
523 * @param s Size in pixels.
525 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
536 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
537 _data[0] = _data[1] = _data[2] = _data[3] = 0;
539 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
540 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
542 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
543 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
545 for (int i = 0; i < components(); ++i) {
546 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
547 _stride[i] = stride_round_up (i, _line_size, _aligned ? 32 : 1);
549 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
550 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
551 Hence on the last pixel of the last line it reads over the end of
552 the actual data by 1 byte. If the width of an image is a multiple
553 of the stride alignment there will be no padding at the end of image lines.
554 OS X crashes on this illegal read, though other operating systems don't
555 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
556 for that instruction to read safely.
558 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
559 over-reads by more then _avx. I can't follow the code to work out how much,
560 so I'll just over-allocate by 32 bytes and have done with it. Empirical
561 testing suggests that it works.
563 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * lines (i) + 32);
567 Image::Image (Image const & other)
569 , _pixel_format (other._pixel_format)
570 , _aligned (other._aligned)
574 for (int i = 0; i < components(); ++i) {
575 uint8_t* p = _data[i];
576 uint8_t* q = other._data[i];
577 for (int j = 0; j < lines(i); ++j) {
578 memcpy (p, q, _line_size[i]);
580 q += other.stride()[i];
585 Image::Image (AVFrame* frame)
586 : dcp::Image (dcp::Size (frame->width, frame->height))
587 , _pixel_format (static_cast<AVPixelFormat> (frame->format))
592 for (int i = 0; i < components(); ++i) {
593 uint8_t* p = _data[i];
594 uint8_t* q = frame->data[i];
595 for (int j = 0; j < lines(i); ++j) {
596 memcpy (p, q, _line_size[i]);
598 /* AVFrame's linesize is what we call `stride' */
599 q += frame->linesize[i];
604 Image::Image (shared_ptr<const Image> other, bool aligned)
606 , _pixel_format (other->_pixel_format)
611 for (int i = 0; i < components(); ++i) {
612 DCPOMATIC_ASSERT (line_size()[i] == other->line_size()[i]);
613 uint8_t* p = _data[i];
614 uint8_t* q = other->data()[i];
615 for (int j = 0; j < lines(i); ++j) {
616 memcpy (p, q, line_size()[i]);
618 q += other->stride()[i];
624 Image::operator= (Image const & other)
626 if (this == &other) {
636 Image::swap (Image & other)
638 dcp::Image::swap (other);
640 std::swap (_pixel_format, other._pixel_format);
642 for (int i = 0; i < 4; ++i) {
643 std::swap (_data[i], other._data[i]);
644 std::swap (_line_size[i], other._line_size[i]);
645 std::swap (_stride[i], other._stride[i]);
648 std::swap (_aligned, other._aligned);
651 /** Destroy a Image */
654 for (int i = 0; i < components(); ++i) {
659 av_free (_line_size);
670 Image::line_size () const
676 Image::stride () const
688 Image::aligned () const
694 merge (list<PositionImage> images)
696 if (images.empty ()) {
697 return PositionImage ();
700 if (images.size() == 1) {
701 return images.front ();
704 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
705 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
706 all.extend (dcpomatic::Rect<int> (i->position, i->image->size().width, i->image->size().height));
709 shared_ptr<Image> merged (new Image (images.front().image->pixel_format (), dcp::Size (all.width, all.height), true));
710 merged->make_transparent ();
711 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
712 merged->alpha_blend (i->image, i->position - all.position());
715 return PositionImage (merged, all.position ());
719 Image::digest () const
721 MD5Digester digester;
723 for (int i = 0; i < components(); ++i) {
724 digester.add (data()[i], line_size()[i]);
727 return digester.get ();
731 operator== (Image const & a, Image const & b)
733 if (a.components() != b.components() || a.pixel_format() != b.pixel_format() || a.aligned() != b.aligned()) {
737 for (int c = 0; c < a.components(); ++c) {
738 if (a.lines(c) != b.lines(c) || a.line_size()[c] != b.line_size()[c] || a.stride()[c] != b.stride()[c]) {
742 uint8_t* p = a.data()[c];
743 uint8_t* q = b.data()[c];
744 for (int y = 0; y < a.lines(c); ++y) {
745 if (memcmp (p, q, a.line_size()[c]) != 0) {
758 Image::fade (float f)
760 switch (_pixel_format) {
761 case PIX_FMT_YUV420P:
762 case PIX_FMT_YUV422P:
763 case PIX_FMT_YUV444P:
764 case PIX_FMT_YUV411P:
765 case PIX_FMT_YUVJ420P:
766 case PIX_FMT_YUVJ422P:
767 case PIX_FMT_YUVJ444P:
773 case PIX_FMT_RGB555LE:
775 for (int c = 0; c < 3; ++c) {
776 uint8_t* p = data()[c];
777 for (int y = 0; y < lines(c); ++y) {
779 for (int x = 0; x < line_size()[c]; ++x) {
780 *q = int (float (*q) * f);
788 case PIX_FMT_YUV422P9LE:
789 case PIX_FMT_YUV444P9LE:
790 case PIX_FMT_YUV422P10LE:
791 case PIX_FMT_YUV444P10LE:
792 case PIX_FMT_YUV422P16LE:
793 case PIX_FMT_YUV444P16LE:
794 case AV_PIX_FMT_YUVA420P9LE:
795 case AV_PIX_FMT_YUVA422P9LE:
796 case AV_PIX_FMT_YUVA444P9LE:
797 case AV_PIX_FMT_YUVA420P10LE:
798 case AV_PIX_FMT_YUVA422P10LE:
799 case AV_PIX_FMT_YUVA444P10LE:
800 /* 16-bit little-endian */
801 for (int c = 0; c < 3; ++c) {
802 int const stride_pixels = stride()[c] / 2;
803 int const line_size_pixels = line_size()[c] / 2;
804 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
805 for (int y = 0; y < lines(c); ++y) {
807 for (int x = 0; x < line_size_pixels; ++x) {
808 *q = int (float (*q) * f);
816 case PIX_FMT_YUV422P9BE:
817 case PIX_FMT_YUV444P9BE:
818 case PIX_FMT_YUV444P10BE:
819 case PIX_FMT_YUV422P10BE:
820 case AV_PIX_FMT_YUVA420P9BE:
821 case AV_PIX_FMT_YUVA422P9BE:
822 case AV_PIX_FMT_YUVA444P9BE:
823 case AV_PIX_FMT_YUVA420P10BE:
824 case AV_PIX_FMT_YUVA422P10BE:
825 case AV_PIX_FMT_YUVA444P10BE:
826 case AV_PIX_FMT_YUVA420P16BE:
827 case AV_PIX_FMT_YUVA422P16BE:
828 case AV_PIX_FMT_YUVA444P16BE:
829 /* 16-bit big-endian */
830 for (int c = 0; c < 3; ++c) {
831 int const stride_pixels = stride()[c] / 2;
832 int const line_size_pixels = line_size()[c] / 2;
833 uint16_t* p = reinterpret_cast<uint16_t*> (data()[c]);
834 for (int y = 0; y < lines(c); ++y) {
836 for (int x = 0; x < line_size_pixels; ++x) {
837 *q = swap_16 (int (float (swap_16 (*q)) * f));
845 case PIX_FMT_UYVY422:
847 int const Y = lines(0);
848 int const X = line_size()[0];
849 uint8_t* p = data()[0];
850 for (int y = 0; y < Y; ++y) {
851 for (int x = 0; x < X; ++x) {
852 *p = int (float (*p) * f);
860 throw PixelFormatError ("fade()", _pixel_format);