2 Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /** @file src/image.cc
21 * @brief A class to describe a video image.
26 #include <libswscale/swscale.h>
27 #include <libavutil/pixfmt.h>
28 #include <libavutil/pixdesc.h>
31 #include "exceptions.h"
35 #include "md5_digester.h"
44 using std::stringstream;
45 using boost::shared_ptr;
49 Image::line_factor (int n) const
55 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
57 throw PixelFormatError ("lines()", _pixel_format);
60 return pow (2.0f, d->log2_chroma_h);
63 /** @param n Component index.
64 * @return Number of lines in the image for the given component.
67 Image::lines (int n) const
69 return rint (ceil (static_cast<double>(size().height) / line_factor (n)));
72 /** @return Number of components */
74 Image::components () const
76 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
78 throw PixelFormatError ("components()", _pixel_format);
81 if ((d->flags & PIX_FMT_PLANAR) == 0) {
85 return d->nb_components;
88 /** Crop this image, scale it to `inter_size' and then place it in a black frame of `out_size' */
90 Image::crop_scale_window (Crop crop, dcp::Size inter_size, dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
93 /* Empirical testing suggests that sws_scale() will crash if
94 the input image is not aligned.
98 assert (out_size.width >= inter_size.width);
99 assert (out_size.height >= inter_size.height);
101 /* Here's an image of out_size */
102 shared_ptr<Image> out (new Image (out_format, out_size, out_aligned));
105 /* Size of the image after any crop */
106 dcp::Size const cropped_size = crop.apply (size ());
108 /* Scale context for a scale from cropped_size to inter_size */
109 struct SwsContext* scale_context = sws_getContext (
110 cropped_size.width, cropped_size.height, pixel_format(),
111 inter_size.width, inter_size.height, out_format,
112 scaler->ffmpeg_id (), 0, 0, 0
115 if (!scale_context) {
116 throw StringError (N_("Could not allocate SwsContext"));
119 /* Prepare input data pointers with crop */
120 uint8_t* scale_in_data[components()];
121 for (int c = 0; c < components(); ++c) {
122 scale_in_data[c] = data()[c] + int (rint (bytes_per_pixel(c) * crop.left)) + stride()[c] * (crop.top / line_factor(c));
125 /* Corner of the image within out_size */
126 Position<int> const corner ((out_size.width - inter_size.width) / 2, (out_size.height - inter_size.height) / 2);
128 uint8_t* scale_out_data[out->components()];
129 for (int c = 0; c < out->components(); ++c) {
130 scale_out_data[c] = out->data()[c] + int (rint (out->bytes_per_pixel(c) * corner.x)) + out->stride()[c] * corner.y;
135 scale_in_data, stride(),
136 0, cropped_size.height,
137 scale_out_data, out->stride()
140 sws_freeContext (scale_context);
146 Image::scale (dcp::Size out_size, Scaler const * scaler, AVPixelFormat out_format, bool out_aligned) const
149 /* Empirical testing suggests that sws_scale() will crash if
150 the input image is not aligned.
154 shared_ptr<Image> scaled (new Image (out_format, out_size, out_aligned));
156 struct SwsContext* scale_context = sws_getContext (
157 size().width, size().height, pixel_format(),
158 out_size.width, out_size.height, out_format,
159 scaler->ffmpeg_id (), 0, 0, 0
166 scaled->data(), scaled->stride()
169 sws_freeContext (scale_context);
175 Image::crop (Crop crop, bool aligned) const
177 dcp::Size cropped_size = crop.apply (size ());
178 shared_ptr<Image> out (new Image (pixel_format(), cropped_size, aligned));
180 for (int c = 0; c < components(); ++c) {
181 int const crop_left_in_bytes = bytes_per_pixel(c) * crop.left;
182 /* bytes_per_pixel() could be a fraction; in this case the stride will be rounded
183 up, and we need to make sure that we copy over the width (up to the stride)
184 rather than short of the width; hence the ceil() here.
186 int const cropped_width_in_bytes = ceil (bytes_per_pixel(c) * cropped_size.width);
188 /* Start of the source line, cropped from the top but not the left */
189 uint8_t* in_p = data()[c] + (crop.top / out->line_factor(c)) * stride()[c];
190 uint8_t* out_p = out->data()[c];
192 for (int y = 0; y < out->lines(c); ++y) {
193 memcpy (out_p, in_p + crop_left_in_bytes, cropped_width_in_bytes);
195 out_p += out->stride()[c];
202 /** Blacken a YUV image whose bits per pixel is rounded up to 16 */
204 Image::yuv_16_black (uint16_t v, bool alpha)
206 memset (data()[0], 0, lines(0) * stride()[0]);
207 for (int i = 1; i < 3; ++i) {
208 int16_t* p = reinterpret_cast<int16_t*> (data()[i]);
209 for (int y = 0; y < lines(i); ++y) {
210 /* We divide by 2 here because we are writing 2 bytes at a time */
211 for (int x = 0; x < line_size()[i] / 2; ++x) {
214 p += stride()[i] / 2;
219 memset (data()[3], 0, lines(3) * stride()[3]);
224 Image::swap_16 (uint16_t v)
226 return ((v >> 8) & 0xff) | ((v & 0xff) << 8);
232 /* U/V black value for 8-bit colour */
233 static uint8_t const eight_bit_uv = (1 << 7) - 1;
234 /* U/V black value for 9-bit colour */
235 static uint16_t const nine_bit_uv = (1 << 8) - 1;
236 /* U/V black value for 10-bit colour */
237 static uint16_t const ten_bit_uv = (1 << 9) - 1;
238 /* U/V black value for 16-bit colour */
239 static uint16_t const sixteen_bit_uv = (1 << 15) - 1;
241 switch (_pixel_format) {
242 case PIX_FMT_YUV420P:
243 case PIX_FMT_YUV422P:
244 case PIX_FMT_YUV444P:
245 case PIX_FMT_YUV411P:
246 memset (data()[0], 0, lines(0) * stride()[0]);
247 memset (data()[1], eight_bit_uv, lines(1) * stride()[1]);
248 memset (data()[2], eight_bit_uv, lines(2) * stride()[2]);
251 case PIX_FMT_YUVJ420P:
252 case PIX_FMT_YUVJ422P:
253 case PIX_FMT_YUVJ444P:
254 memset (data()[0], 0, lines(0) * stride()[0]);
255 memset (data()[1], eight_bit_uv + 1, lines(1) * stride()[1]);
256 memset (data()[2], eight_bit_uv + 1, lines(2) * stride()[2]);
259 case PIX_FMT_YUV422P9LE:
260 case PIX_FMT_YUV444P9LE:
261 yuv_16_black (nine_bit_uv, false);
264 case PIX_FMT_YUV422P9BE:
265 case PIX_FMT_YUV444P9BE:
266 yuv_16_black (swap_16 (nine_bit_uv), false);
269 case PIX_FMT_YUV422P10LE:
270 case PIX_FMT_YUV444P10LE:
271 yuv_16_black (ten_bit_uv, false);
274 case PIX_FMT_YUV422P16LE:
275 case PIX_FMT_YUV444P16LE:
276 yuv_16_black (sixteen_bit_uv, false);
279 case PIX_FMT_YUV444P10BE:
280 case PIX_FMT_YUV422P10BE:
281 yuv_16_black (swap_16 (ten_bit_uv), false);
284 case AV_PIX_FMT_YUVA420P9BE:
285 case AV_PIX_FMT_YUVA422P9BE:
286 case AV_PIX_FMT_YUVA444P9BE:
287 yuv_16_black (swap_16 (nine_bit_uv), true);
290 case AV_PIX_FMT_YUVA420P9LE:
291 case AV_PIX_FMT_YUVA422P9LE:
292 case AV_PIX_FMT_YUVA444P9LE:
293 yuv_16_black (nine_bit_uv, true);
296 case AV_PIX_FMT_YUVA420P10BE:
297 case AV_PIX_FMT_YUVA422P10BE:
298 case AV_PIX_FMT_YUVA444P10BE:
299 yuv_16_black (swap_16 (ten_bit_uv), true);
302 case AV_PIX_FMT_YUVA420P10LE:
303 case AV_PIX_FMT_YUVA422P10LE:
304 case AV_PIX_FMT_YUVA444P10LE:
305 yuv_16_black (ten_bit_uv, true);
308 case AV_PIX_FMT_YUVA420P16BE:
309 case AV_PIX_FMT_YUVA422P16BE:
310 case AV_PIX_FMT_YUVA444P16BE:
311 yuv_16_black (swap_16 (sixteen_bit_uv), true);
314 case AV_PIX_FMT_YUVA420P16LE:
315 case AV_PIX_FMT_YUVA422P16LE:
316 case AV_PIX_FMT_YUVA444P16LE:
317 yuv_16_black (sixteen_bit_uv, true);
325 memset (data()[0], 0, lines(0) * stride()[0]);
328 case PIX_FMT_UYVY422:
330 int const Y = lines(0);
331 int const X = line_size()[0];
332 uint8_t* p = data()[0];
333 for (int y = 0; y < Y; ++y) {
334 for (int x = 0; x < X / 4; ++x) {
335 *p++ = eight_bit_uv; // Cb
337 *p++ = eight_bit_uv; // Cr
345 throw PixelFormatError ("make_black()", _pixel_format);
350 Image::make_transparent ()
352 if (_pixel_format != PIX_FMT_RGBA) {
353 throw PixelFormatError ("make_transparent()", _pixel_format);
356 memset (data()[0], 0, lines(0) * stride()[0]);
360 Image::alpha_blend (shared_ptr<const Image> other, Position<int> position)
365 if (_pixel_format == PIX_FMT_BGRA && other->pixel_format() == PIX_FMT_RGBA) {
368 } else if (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGBA) {
375 int start_tx = position.x;
379 start_ox = -start_tx;
383 int start_ty = position.y;
387 start_oy = -start_ty;
391 for (int ty = start_ty, oy = start_oy; ty < size().height && oy < other->size().height; ++ty, ++oy) {
392 uint8_t* tp = data()[0] + ty * stride()[0] + position.x * this_bpp;
393 uint8_t* op = other->data()[0] + oy * other->stride()[0];
394 for (int tx = start_tx, ox = start_ox; tx < size().width && ox < other->size().width; ++tx, ++ox) {
395 float const alpha = float (op[3]) / 255;
396 tp[0] = (tp[0] * (1 - alpha)) + op[0] * alpha;
397 tp[1] = (tp[1] * (1 - alpha)) + op[1] * alpha;
398 tp[2] = (tp[2] * (1 - alpha)) + op[2] * alpha;
406 Image::copy (shared_ptr<const Image> other, Position<int> position)
408 /* Only implemented for RGB24 onto RGB24 so far */
409 assert (_pixel_format == PIX_FMT_RGB24 && other->pixel_format() == PIX_FMT_RGB24);
410 assert (position.x >= 0 && position.y >= 0);
412 int const N = min (position.x + other->size().width, size().width) - position.x;
413 for (int ty = position.y, oy = 0; ty < size().height && oy < other->size().height; ++ty, ++oy) {
414 uint8_t * const tp = data()[0] + ty * stride()[0] + position.x * 3;
415 uint8_t * const op = other->data()[0] + oy * other->stride()[0];
416 memcpy (tp, op, N * 3);
421 Image::read_from_socket (shared_ptr<Socket> socket)
423 for (int i = 0; i < components(); ++i) {
424 uint8_t* p = data()[i];
425 for (int y = 0; y < lines(i); ++y) {
426 socket->read (p, line_size()[i]);
433 Image::write_to_socket (shared_ptr<Socket> socket) const
435 for (int i = 0; i < components(); ++i) {
436 uint8_t* p = data()[i];
437 for (int y = 0; y < lines(i); ++y) {
438 socket->write (p, line_size()[i]);
446 Image::bytes_per_pixel (int c) const
448 AVPixFmtDescriptor const * d = av_pix_fmt_desc_get(_pixel_format);
450 throw PixelFormatError ("lines()", _pixel_format);
453 if (c >= components()) {
457 float bpp[4] = { 0, 0, 0, 0 };
459 bpp[0] = floor ((d->comp[0].depth_minus1 + 1 + 7) / 8);
460 if (d->nb_components > 1) {
461 bpp[1] = floor ((d->comp[1].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
463 if (d->nb_components > 2) {
464 bpp[2] = floor ((d->comp[2].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
466 if (d->nb_components > 3) {
467 bpp[3] = floor ((d->comp[3].depth_minus1 + 1 + 7) / 8) / pow (2.0f, d->log2_chroma_w);
470 if ((d->flags & PIX_FMT_PLANAR) == 0) {
471 /* Not planar; sum them up */
472 return bpp[0] + bpp[1] + bpp[2] + bpp[3];
478 /** Construct a Image of a given size and format, allocating memory
481 * @param p Pixel format.
482 * @param s Size in pixels.
484 Image::Image (AVPixelFormat p, dcp::Size s, bool aligned)
495 _data = (uint8_t **) wrapped_av_malloc (4 * sizeof (uint8_t *));
496 _data[0] = _data[1] = _data[2] = _data[3] = 0;
498 _line_size = (int *) wrapped_av_malloc (4 * sizeof (int));
499 _line_size[0] = _line_size[1] = _line_size[2] = _line_size[3] = 0;
501 _stride = (int *) wrapped_av_malloc (4 * sizeof (int));
502 _stride[0] = _stride[1] = _stride[2] = _stride[3] = 0;
504 for (int i = 0; i < components(); ++i) {
505 _line_size[i] = ceil (_size.width * bytes_per_pixel(i));
506 _stride[i] = stride_round_up (i, _line_size, _aligned ? 32 : 1);
508 /* The assembler function ff_rgb24ToY_avx (in libswscale/x86/input.asm)
509 uses a 16-byte fetch to read three bytes (R/G/B) of image data.
510 Hence on the last pixel of the last line it reads over the end of
511 the actual data by 1 byte. If the width of an image is a multiple
512 of the stride alignment there will be no padding at the end of image lines.
513 OS X crashes on this illegal read, though other operating systems don't
514 seem to mind. The nasty + 1 in this malloc makes sure there is always a byte
515 for that instruction to read safely.
517 Further to the above, valgrind is now telling me that ff_rgb24ToY_ssse3
518 over-reads by more then _avx. I can't follow the code to work out how much,
519 so I'll just over-allocate by 32 bytes and have done with it. Empirical
520 testing suggests that it works.
522 _data[i] = (uint8_t *) wrapped_av_malloc (_stride[i] * lines (i) + 32);
526 Image::Image (Image const & other)
528 , _pixel_format (other._pixel_format)
529 , _aligned (other._aligned)
533 for (int i = 0; i < components(); ++i) {
534 uint8_t* p = _data[i];
535 uint8_t* q = other._data[i];
536 for (int j = 0; j < lines(i); ++j) {
537 memcpy (p, q, _line_size[i]);
539 q += other.stride()[i];
544 Image::Image (AVFrame* frame)
545 : dcp::Image (dcp::Size (frame->width, frame->height))
546 , _pixel_format (static_cast<AVPixelFormat> (frame->format))
551 for (int i = 0; i < components(); ++i) {
552 uint8_t* p = _data[i];
553 uint8_t* q = frame->data[i];
554 for (int j = 0; j < lines(i); ++j) {
555 memcpy (p, q, _line_size[i]);
557 /* AVFrame's linesize is what we call `stride' */
558 q += frame->linesize[i];
563 Image::Image (shared_ptr<const Image> other, bool aligned)
565 , _pixel_format (other->_pixel_format)
570 for (int i = 0; i < components(); ++i) {
571 assert(line_size()[i] == other->line_size()[i]);
572 uint8_t* p = _data[i];
573 uint8_t* q = other->data()[i];
574 for (int j = 0; j < lines(i); ++j) {
575 memcpy (p, q, line_size()[i]);
577 q += other->stride()[i];
583 Image::operator= (Image const & other)
585 if (this == &other) {
595 Image::swap (Image & other)
597 dcp::Image::swap (other);
599 std::swap (_pixel_format, other._pixel_format);
601 for (int i = 0; i < 4; ++i) {
602 std::swap (_data[i], other._data[i]);
603 std::swap (_line_size[i], other._line_size[i]);
604 std::swap (_stride[i], other._stride[i]);
607 std::swap (_aligned, other._aligned);
610 /** Destroy a Image */
613 for (int i = 0; i < components(); ++i) {
618 av_free (_line_size);
629 Image::line_size () const
635 Image::stride () const
647 Image::aligned () const
653 merge (list<PositionImage> images)
655 if (images.empty ()) {
656 return PositionImage ();
659 dcpomatic::Rect<int> all (images.front().position, images.front().image->size().width, images.front().image->size().height);
660 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
661 all.extend (dcpomatic::Rect<int> (i->position, i->image->size().width, i->image->size().height));
664 shared_ptr<Image> merged (new Image (images.front().image->pixel_format (), dcp::Size (all.width, all.height), true));
665 merged->make_transparent ();
666 for (list<PositionImage>::const_iterator i = images.begin(); i != images.end(); ++i) {
667 merged->alpha_blend (i->image, i->position);
670 return PositionImage (merged, all.position ());
674 Image::digest () const
676 MD5Digester digester;
678 for (int i = 0; i < components(); ++i) {
679 digester.add (data()[i], line_size()[i]);
682 return digester.get ();