2 Copyright (C) 2012-2018 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
21 /** @file src/ffmpeg_decoder.cc
22 * @brief A decoder using FFmpeg to decode content.
26 #include "exceptions.h"
30 #include "dcpomatic_log.h"
31 #include "ffmpeg_decoder.h"
32 #include "text_decoder.h"
33 #include "ffmpeg_audio_stream.h"
34 #include "ffmpeg_subtitle_stream.h"
35 #include "video_filter_graph.h"
36 #include "audio_buffers.h"
37 #include "ffmpeg_content.h"
38 #include "raw_image_proxy.h"
39 #include "video_decoder.h"
41 #include "audio_decoder.h"
42 #include "compose.hpp"
43 #include "text_content.h"
44 #include "audio_content.h"
45 #include "frame_interval_checker.h"
46 #include <dcp/subtitle_string.h>
47 #include <sub/ssa_reader.h>
48 #include <sub/subtitle.h>
49 #include <sub/collect.h>
51 #include <libavcodec/avcodec.h>
52 #include <libavformat/avformat.h>
54 #include <boost/foreach.hpp>
55 #include <boost/algorithm/string.hpp>
71 using boost::shared_ptr;
72 using boost::is_any_of;
74 using boost::optional;
75 using boost::dynamic_pointer_cast;
77 using namespace dcpomatic;
79 FFmpegDecoder::FFmpegDecoder (shared_ptr<const Film> film, shared_ptr<const FFmpegContent> c, bool fast)
82 , _have_current_subtitle (false)
84 if (c->video && c->video->use()) {
85 video.reset (new VideoDecoder (this, c));
86 _pts_offset = pts_offset (c->ffmpeg_audio_streams(), c->first_video(), c->active_video_frame_rate(film));
87 /* It doesn't matter what size or pixel format this is, it just needs to be black */
88 _black_image.reset (new Image (AV_PIX_FMT_RGB24, dcp::Size (128, 128), true));
89 _black_image->make_black ();
91 _pts_offset = ContentTime ();
95 audio.reset (new AudioDecoder (this, c->audio, fast));
99 /* XXX: this time here should be the time of the first subtitle, not 0 */
100 text.push_back (shared_ptr<TextDecoder> (new TextDecoder (this, c->only_text(), ContentTime())));
103 _next_time.resize (_format_context->nb_streams);
107 FFmpegDecoder::flush ()
109 /* Get any remaining frames */
114 /* XXX: should we reset _packet.data and size after each *_decode_* call? */
116 while (video && decode_video_packet()) {}
119 decode_audio_packet ();
122 /* Make sure all streams are the same length and round up to the next video frame */
124 FrameRateChange const frc = film()->active_frame_rate_change(_ffmpeg_content->position());
125 ContentTime full_length (_ffmpeg_content->full_length(film()), frc);
126 full_length = full_length.ceil (frc.source);
128 double const vfr = _ffmpeg_content->video_frame_rate().get();
129 Frame const f = full_length.frames_round (vfr);
130 Frame v = video->position(film()).get_value_or(ContentTime()).frames_round(vfr) + 1;
132 video->emit (film(), shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)), v);
137 BOOST_FOREACH (shared_ptr<FFmpegAudioStream> i, _ffmpeg_content->ffmpeg_audio_streams ()) {
138 ContentTime a = audio->stream_position(film(), i);
139 /* Unfortunately if a is 0 that really means that we don't know the stream position since
140 there has been no data on it since the last seek. In this case we'll just do nothing
141 here. I'm not sure if that's the right idea.
143 if (a > ContentTime()) {
144 while (a < full_length) {
145 ContentTime to_do = min (full_length - a, ContentTime::from_seconds (0.1));
146 shared_ptr<AudioBuffers> silence (new AudioBuffers (i->channels(), to_do.frames_ceil (i->frame_rate())));
147 silence->make_silent ();
148 audio->emit (film(), i, silence, a);
160 FFmpegDecoder::pass ()
162 #ifdef DCPOMATIC_VARIANT_SWAROOP
163 if (_ffmpeg_content->encrypted() && !_ffmpeg_content->kdm()) {
168 int r = av_read_frame (_format_context, &_packet);
170 /* AVERROR_INVALIDDATA can apparently be returned sometimes even when av_read_frame
171 has pretty-much succeeded (and hence generated data which should be processed).
172 Hence it makes sense to continue here in that case.
174 if (r < 0 && r != AVERROR_INVALIDDATA) {
175 if (r != AVERROR_EOF) {
176 /* Maybe we should fail here, but for now we'll just finish off instead */
178 av_strerror (r, buf, sizeof(buf));
179 LOG_ERROR (N_("error on av_read_frame (%1) (%2)"), &buf[0], r);
186 int const si = _packet.stream_index;
187 shared_ptr<const FFmpegContent> fc = _ffmpeg_content;
189 if (_video_stream && si == _video_stream.get() && video && !video->ignore()) {
190 decode_video_packet ();
191 } else if (fc->subtitle_stream() && fc->subtitle_stream()->uses_index(_format_context, si) && !only_text()->ignore()) {
192 decode_subtitle_packet ();
194 decode_audio_packet ();
197 av_packet_unref (&_packet);
201 /** @param data pointer to array of pointers to buffers.
202 * Only the first buffer will be used for non-planar data, otherwise there will be one per channel.
204 shared_ptr<AudioBuffers>
205 FFmpegDecoder::deinterleave_audio (shared_ptr<FFmpegAudioStream> stream) const
207 DCPOMATIC_ASSERT (bytes_per_audio_sample (stream));
209 DCPOMATIC_DISABLE_WARNINGS
210 int const size = av_samples_get_buffer_size (
211 0, stream->stream(_format_context)->codec->channels, _frame->nb_samples, audio_sample_format (stream), 1
213 DCPOMATIC_ENABLE_WARNINGS
215 /* XXX: can't we just use _frame->nb_samples directly here? */
216 /* XXX: can't we use swr_convert() to do the format conversion? */
218 /* Deinterleave and convert to float */
220 /* total_samples and frames will be rounded down here, so if there are stray samples at the end
221 of the block that do not form a complete sample or frame they will be dropped.
223 int const total_samples = size / bytes_per_audio_sample (stream);
224 int const channels = stream->channels();
225 int const frames = total_samples / channels;
226 shared_ptr<AudioBuffers> audio (new AudioBuffers (channels, frames));
227 float** data = audio->data();
229 switch (audio_sample_format (stream)) {
230 case AV_SAMPLE_FMT_U8:
232 uint8_t* p = reinterpret_cast<uint8_t *> (_frame->data[0]);
235 for (int i = 0; i < total_samples; ++i) {
236 data[channel][sample] = float(*p++) / (1 << 23);
239 if (channel == channels) {
247 case AV_SAMPLE_FMT_S16:
249 int16_t* p = reinterpret_cast<int16_t *> (_frame->data[0]);
252 for (int i = 0; i < total_samples; ++i) {
253 data[channel][sample] = float(*p++) / (1 << 15);
256 if (channel == channels) {
264 case AV_SAMPLE_FMT_S16P:
266 int16_t** p = reinterpret_cast<int16_t **> (_frame->data);
267 for (int i = 0; i < channels; ++i) {
268 for (int j = 0; j < frames; ++j) {
269 data[i][j] = static_cast<float>(p[i][j]) / (1 << 15);
275 case AV_SAMPLE_FMT_S32:
277 int32_t* p = reinterpret_cast<int32_t *> (_frame->data[0]);
280 for (int i = 0; i < total_samples; ++i) {
281 data[channel][sample] = static_cast<float>(*p++) / 2147483648;
284 if (channel == channels) {
292 case AV_SAMPLE_FMT_S32P:
294 int32_t** p = reinterpret_cast<int32_t **> (_frame->data);
295 for (int i = 0; i < channels; ++i) {
296 for (int j = 0; j < frames; ++j) {
297 data[i][j] = static_cast<float>(p[i][j]) / 2147483648;
303 case AV_SAMPLE_FMT_FLT:
305 float* p = reinterpret_cast<float*> (_frame->data[0]);
308 for (int i = 0; i < total_samples; ++i) {
309 data[channel][sample] = *p++;
312 if (channel == channels) {
320 case AV_SAMPLE_FMT_FLTP:
322 float** p = reinterpret_cast<float**> (_frame->data);
323 DCPOMATIC_ASSERT (_frame->channels <= channels);
324 /* Sometimes there aren't as many channels in the _frame as in the stream */
325 for (int i = 0; i < _frame->channels; ++i) {
326 memcpy (data[i], p[i], frames * sizeof(float));
328 for (int i = _frame->channels; i < channels; ++i) {
329 audio->make_silent (i);
335 throw DecodeError (String::compose (_("Unrecognised audio sample format (%1)"), static_cast<int> (audio_sample_format (stream))));
342 FFmpegDecoder::audio_sample_format (shared_ptr<FFmpegAudioStream> stream) const
344 DCPOMATIC_DISABLE_WARNINGS
345 return stream->stream (_format_context)->codec->sample_fmt;
346 DCPOMATIC_ENABLE_WARNINGS
350 FFmpegDecoder::bytes_per_audio_sample (shared_ptr<FFmpegAudioStream> stream) const
352 return av_get_bytes_per_sample (audio_sample_format (stream));
356 FFmpegDecoder::seek (ContentTime time, bool accurate)
358 Decoder::seek (time, accurate);
360 /* If we are doing an `accurate' seek, we need to use pre-roll, as
361 we don't really know what the seek will give us.
364 ContentTime pre_roll = accurate ? ContentTime::from_seconds (2) : ContentTime (0);
367 /* XXX: it seems debatable whether PTS should be used here...
368 http://www.mjbshaw.com/2012/04/seeking-in-ffmpeg-know-your-timestamp.html
371 optional<int> stream;
374 stream = _video_stream;
376 DCPOMATIC_ASSERT (_ffmpeg_content->audio);
377 shared_ptr<FFmpegAudioStream> s = dynamic_pointer_cast<FFmpegAudioStream> (_ffmpeg_content->audio->stream ());
379 stream = s->index (_format_context);
383 DCPOMATIC_ASSERT (stream);
385 ContentTime u = time - _pts_offset;
386 if (u < ContentTime ()) {
392 u.seconds() / av_q2d (_format_context->streams[stream.get()]->time_base),
397 /* Force re-creation of filter graphs to reset them and hence to make sure
398 they don't have any pre-seek frames knocking about.
400 boost::mutex::scoped_lock lm (_filter_graphs_mutex);
401 _filter_graphs.clear ();
404 if (video_codec_context ()) {
405 avcodec_flush_buffers (video_codec_context());
408 DCPOMATIC_DISABLE_WARNINGS
409 BOOST_FOREACH (shared_ptr<FFmpegAudioStream> i, ffmpeg_content()->ffmpeg_audio_streams()) {
410 avcodec_flush_buffers (i->stream(_format_context)->codec);
412 DCPOMATIC_ENABLE_WARNINGS
414 if (subtitle_codec_context ()) {
415 avcodec_flush_buffers (subtitle_codec_context ());
418 _have_current_subtitle = false;
422 FFmpegDecoder::decode_audio_packet ()
424 /* Audio packets can contain multiple frames, so we may have to call avcodec_decode_audio4
428 AVPacket copy_packet = _packet;
429 int const stream_index = copy_packet.stream_index;
431 /* XXX: inefficient */
432 vector<shared_ptr<FFmpegAudioStream> > streams = ffmpeg_content()->ffmpeg_audio_streams ();
433 vector<shared_ptr<FFmpegAudioStream> >::const_iterator stream = streams.begin ();
434 while (stream != streams.end () && !(*stream)->uses_index (_format_context, stream_index)) {
438 if (stream == streams.end ()) {
439 /* The packet's stream may not be an audio one; just ignore it in this method if so */
443 DCPOMATIC_DISABLE_WARNINGS
444 while (copy_packet.size > 0) {
447 int decode_result = avcodec_decode_audio4 ((*stream)->stream (_format_context)->codec, _frame, &frame_finished, ©_packet);
448 if (decode_result < 0) {
449 /* avcodec_decode_audio4 can sometimes return an error even though it has decoded
450 some valid data; for example dca_subframe_footer can return AVERROR_INVALIDDATA
451 if it overreads the auxiliary data. ffplay carries on if frame_finished is true,
452 even in the face of such an error, so I think we should too.
454 Returning from the method here caused mantis #352.
456 LOG_WARNING ("avcodec_decode_audio4 failed (%1)", decode_result);
458 /* Fudge decode_result so that we come out of the while loop when
459 we've processed this data.
461 decode_result = copy_packet.size;
464 if (frame_finished) {
465 shared_ptr<AudioBuffers> data = deinterleave_audio (*stream);
468 if (_frame->pts == AV_NOPTS_VALUE && _next_time[stream_index]) {
469 /* In some streams we see not every frame coming through with a timestamp; for those
470 that have AV_NOPTS_VALUE we need to work out the timestamp ourselves. This is
471 particularly noticeable with TrueHD streams (see #1111).
473 ct = *_next_time[stream_index];
475 ct = ContentTime::from_seconds (
476 av_frame_get_best_effort_timestamp (_frame) *
477 av_q2d ((*stream)->stream (_format_context)->time_base))
481 _next_time[stream_index] = ct + ContentTime::from_frames(data->frames(), (*stream)->frame_rate());
483 if (ct < ContentTime ()) {
484 /* Discard audio data that comes before time 0 */
485 Frame const remove = min (int64_t (data->frames()), (-ct).frames_ceil(double((*stream)->frame_rate ())));
486 data->move (data->frames() - remove, remove, 0);
487 data->set_frames (data->frames() - remove);
488 ct += ContentTime::from_frames (remove, (*stream)->frame_rate ());
491 if (ct < ContentTime()) {
493 "Crazy timestamp %1 for %2 samples in stream %3 packet pts %4 (ts=%5 tb=%6, off=%7)",
496 copy_packet.stream_index,
498 av_frame_get_best_effort_timestamp(_frame),
499 av_q2d((*stream)->stream(_format_context)->time_base),
500 to_string(_pts_offset)
503 DCPOMATIC_ENABLE_WARNINGS
505 /* Give this data provided there is some, and its time is sane */
506 if (ct >= ContentTime() && data->frames() > 0) {
507 audio->emit (film(), *stream, data, ct);
511 copy_packet.data += decode_result;
512 copy_packet.size -= decode_result;
517 FFmpegDecoder::decode_video_packet ()
519 DCPOMATIC_ASSERT (_video_stream);
522 DCPOMATIC_DISABLE_WARNINGS
523 if (avcodec_decode_video2 (video_codec_context(), _frame, &frame_finished, &_packet) < 0 || !frame_finished) {
526 DCPOMATIC_ENABLE_WARNINGS
528 boost::mutex::scoped_lock lm (_filter_graphs_mutex);
530 shared_ptr<VideoFilterGraph> graph;
532 list<shared_ptr<VideoFilterGraph> >::iterator i = _filter_graphs.begin();
533 while (i != _filter_graphs.end() && !(*i)->can_process (dcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format)) {
537 if (i == _filter_graphs.end ()) {
538 dcp::Fraction vfr (lrint(_ffmpeg_content->video_frame_rate().get() * 1000), 1000);
539 graph.reset (new VideoFilterGraph (dcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format, vfr));
540 graph->setup (_ffmpeg_content->filters ());
541 _filter_graphs.push_back (graph);
542 LOG_GENERAL (N_("New graph for %1x%2, pixel format %3"), _frame->width, _frame->height, _frame->format);
547 list<pair<shared_ptr<Image>, int64_t> > images = graph->process (_frame);
549 for (list<pair<shared_ptr<Image>, int64_t> >::iterator i = images.begin(); i != images.end(); ++i) {
551 shared_ptr<Image> image = i->first;
553 if (i->second != AV_NOPTS_VALUE) {
554 double const pts = i->second * av_q2d (_format_context->streams[_video_stream.get()]->time_base) + _pts_offset.seconds ();
558 shared_ptr<ImageProxy> (new RawImageProxy (image)),
559 llrint(pts * _ffmpeg_content->active_video_frame_rate(film()))
562 LOG_WARNING_NC ("Dropping frame without PTS");
570 FFmpegDecoder::decode_subtitle_packet ()
574 if (avcodec_decode_subtitle2 (subtitle_codec_context(), &sub, &got_subtitle, &_packet) < 0 || !got_subtitle) {
578 /* Stop any current subtitle, either at the time it was supposed to stop, or now if now is sooner */
579 if (_have_current_subtitle) {
580 if (_current_subtitle_to) {
581 only_text()->emit_stop (min(*_current_subtitle_to, subtitle_period(sub).from + _pts_offset));
583 only_text()->emit_stop (subtitle_period(sub).from + _pts_offset);
585 _have_current_subtitle = false;
588 if (sub.num_rects <= 0) {
589 /* Nothing new in this subtitle */
593 /* Subtitle PTS (within the source, not taking into account any of the
594 source that we may have chopped off for the DCP).
596 FFmpegSubtitlePeriod sub_period = subtitle_period (sub);
598 from = sub_period.from + _pts_offset;
600 _current_subtitle_to = *sub_period.to + _pts_offset;
602 _current_subtitle_to = optional<ContentTime>();
603 _have_current_subtitle = true;
606 for (unsigned int i = 0; i < sub.num_rects; ++i) {
607 AVSubtitleRect const * rect = sub.rects[i];
609 switch (rect->type) {
612 case SUBTITLE_BITMAP:
613 decode_bitmap_subtitle (rect, from);
616 cout << "XXX: SUBTITLE_TEXT " << rect->text << "\n";
619 decode_ass_subtitle (rect->ass, from);
624 if (_current_subtitle_to) {
625 only_text()->emit_stop (*_current_subtitle_to);
628 avsubtitle_free (&sub);
632 FFmpegDecoder::decode_bitmap_subtitle (AVSubtitleRect const * rect, ContentTime from)
634 /* Note BGRA is expressed little-endian, so the first byte in the word is B, second
635 G, third R, fourth A.
637 shared_ptr<Image> image (new Image (AV_PIX_FMT_BGRA, dcp::Size (rect->w, rect->h), true));
639 #ifdef DCPOMATIC_HAVE_AVSUBTITLERECT_PICT
640 /* Start of the first line in the subtitle */
641 uint8_t* sub_p = rect->pict.data[0];
642 /* sub_p looks up into a BGRA palette which is at rect->pict.data[1];
643 (i.e. first byte B, second G, third R, fourth A)
645 uint8_t const * palette = rect->pict.data[1];
647 /* Start of the first line in the subtitle */
648 uint8_t* sub_p = rect->data[0];
649 /* sub_p looks up into a BGRA palette which is at rect->data[1].
650 (first byte B, second G, third R, fourth A)
652 uint8_t const * palette = rect->data[1];
654 /* And the stream has a map of those palette colours to colours
655 chosen by the user; created a `mapped' palette from those settings.
657 map<RGBA, RGBA> colour_map = ffmpeg_content()->subtitle_stream()->colours ();
658 vector<RGBA> mapped_palette (rect->nb_colors);
659 for (int i = 0; i < rect->nb_colors; ++i) {
660 RGBA c (palette[2], palette[1], palette[0], palette[3]);
661 map<RGBA, RGBA>::const_iterator j = colour_map.find (c);
662 if (j != colour_map.end ()) {
663 mapped_palette[i] = j->second;
665 /* This colour was not found in the FFmpegSubtitleStream's colour map; probably because
666 it is from a project that was created before this stuff was added. Just use the
667 colour straight from the original palette.
669 mapped_palette[i] = c;
674 /* Start of the output data */
675 uint8_t* out_p = image->data()[0];
677 for (int y = 0; y < rect->h; ++y) {
678 uint8_t* sub_line_p = sub_p;
679 uint8_t* out_line_p = out_p;
680 for (int x = 0; x < rect->w; ++x) {
681 RGBA const p = mapped_palette[*sub_line_p++];
687 #ifdef DCPOMATIC_HAVE_AVSUBTITLERECT_PICT
688 sub_p += rect->pict.linesize[0];
690 sub_p += rect->linesize[0];
692 out_p += image->stride()[0];
695 int target_width = subtitle_codec_context()->width;
696 if (target_width == 0 && video_codec_context()) {
697 /* subtitle_codec_context()->width == 0 has been seen in the wild but I don't
698 know if it's supposed to mean something from FFmpeg's point of view.
700 target_width = video_codec_context()->width;
702 int target_height = subtitle_codec_context()->height;
703 if (target_height == 0 && video_codec_context()) {
704 target_height = video_codec_context()->height;
706 DCPOMATIC_ASSERT (target_width);
707 DCPOMATIC_ASSERT (target_height);
708 dcpomatic::Rect<double> const scaled_rect (
709 static_cast<double> (rect->x) / target_width,
710 static_cast<double> (rect->y) / target_height,
711 static_cast<double> (rect->w) / target_width,
712 static_cast<double> (rect->h) / target_height
715 only_text()->emit_bitmap_start (from, image, scaled_rect);
719 FFmpegDecoder::decode_ass_subtitle (string ass, ContentTime from)
721 /* We have no styles and no Format: line, so I'm assuming that FFmpeg
722 produces a single format of Dialogue: lines...
727 for (size_t i = 0; i < ass.length(); ++i) {
728 if (commas < 9 && ass[i] == ',') {
730 } else if (commas == 9) {
739 sub::RawSubtitle base;
740 list<sub::RawSubtitle> raw = sub::SSAReader::parse_line (
743 _ffmpeg_content->video->size().width,
744 _ffmpeg_content->video->size().height
747 BOOST_FOREACH (sub::Subtitle const & i, sub::collect<list<sub::Subtitle> > (raw)) {
748 only_text()->emit_plain_start (from, i);