2 Copyright (C) 2012-2018 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
21 /** @file src/ffmpeg_decoder.cc
22 * @brief A decoder using FFmpeg to decode content.
26 #include "exceptions.h"
30 #include "dcpomatic_log.h"
31 #include "ffmpeg_decoder.h"
32 #include "text_decoder.h"
33 #include "ffmpeg_audio_stream.h"
34 #include "ffmpeg_subtitle_stream.h"
35 #include "video_filter_graph.h"
36 #include "audio_buffers.h"
37 #include "ffmpeg_content.h"
38 #include "raw_image_proxy.h"
39 #include "video_decoder.h"
41 #include "audio_decoder.h"
42 #include "compose.hpp"
43 #include "text_content.h"
44 #include "audio_content.h"
45 #include "frame_interval_checker.h"
46 #include <dcp/subtitle_string.h>
47 #include <sub/ssa_reader.h>
48 #include <sub/subtitle.h>
49 #include <sub/collect.h>
51 #include <libavcodec/avcodec.h>
52 #include <libavformat/avformat.h>
54 #include <boost/foreach.hpp>
55 #include <boost/algorithm/string.hpp>
71 using boost::shared_ptr;
72 using boost::is_any_of;
74 using boost::optional;
75 using boost::dynamic_pointer_cast;
77 using namespace dcpomatic;
79 FFmpegDecoder::FFmpegDecoder (shared_ptr<const Film> film, shared_ptr<const FFmpegContent> c, bool fast)
82 , _have_current_subtitle (false)
84 if (c->video && c->video->use()) {
85 video.reset (new VideoDecoder (this, c));
86 _pts_offset = pts_offset (c->ffmpeg_audio_streams(), c->first_video(), c->active_video_frame_rate(film));
87 /* It doesn't matter what size or pixel format this is, it just needs to be black */
88 _black_image.reset (new Image (AV_PIX_FMT_RGB24, dcp::Size (128, 128), true));
89 _black_image->make_black ();
91 _pts_offset = ContentTime ();
95 audio.reset (new AudioDecoder (this, c->audio, fast));
99 /* XXX: this time here should be the time of the first subtitle, not 0 */
100 text.push_back (shared_ptr<TextDecoder> (new TextDecoder (this, c->only_text(), ContentTime())));
103 _next_time.resize (_format_context->nb_streams);
107 FFmpegDecoder::flush ()
109 /* Get any remaining frames */
114 /* XXX: should we reset _packet.data and size after each *_decode_* call? */
116 while (video && decode_video_packet()) {}
119 decode_audio_packet ();
122 /* Make sure all streams are the same length and round up to the next video frame */
124 FrameRateChange const frc = film()->active_frame_rate_change(_ffmpeg_content->position());
125 ContentTime full_length (_ffmpeg_content->full_length(film()), frc);
126 full_length = full_length.ceil (frc.source);
128 double const vfr = _ffmpeg_content->video_frame_rate().get();
129 Frame const f = full_length.frames_round (vfr);
130 Frame v = video->position(film()).get_value_or(ContentTime()).frames_round(vfr) + 1;
132 video->emit (film(), shared_ptr<const ImageProxy> (new RawImageProxy (_black_image)), v);
137 BOOST_FOREACH (shared_ptr<FFmpegAudioStream> i, _ffmpeg_content->ffmpeg_audio_streams ()) {
138 ContentTime a = audio->stream_position(film(), i);
139 /* Unfortunately if a is 0 that really means that we don't know the stream position since
140 there has been no data on it since the last seek. In this case we'll just do nothing
141 here. I'm not sure if that's the right idea.
143 if (a > ContentTime()) {
144 while (a < full_length) {
145 ContentTime to_do = min (full_length - a, ContentTime::from_seconds (0.1));
146 shared_ptr<AudioBuffers> silence (new AudioBuffers (i->channels(), to_do.frames_ceil (i->frame_rate())));
147 silence->make_silent ();
148 audio->emit (film(), i, silence, a, true);
160 FFmpegDecoder::pass ()
162 int r = av_read_frame (_format_context, &_packet);
164 /* AVERROR_INVALIDDATA can apparently be returned sometimes even when av_read_frame
165 has pretty-much succeeded (and hence generated data which should be processed).
166 Hence it makes sense to continue here in that case.
168 if (r < 0 && r != AVERROR_INVALIDDATA) {
169 if (r != AVERROR_EOF) {
170 /* Maybe we should fail here, but for now we'll just finish off instead */
172 av_strerror (r, buf, sizeof(buf));
173 LOG_ERROR (N_("error on av_read_frame (%1) (%2)"), &buf[0], r);
180 int const si = _packet.stream_index;
181 shared_ptr<const FFmpegContent> fc = _ffmpeg_content;
183 if (_video_stream && si == _video_stream.get() && video && !video->ignore()) {
184 decode_video_packet ();
185 } else if (fc->subtitle_stream() && fc->subtitle_stream()->uses_index(_format_context, si) && !only_text()->ignore()) {
186 decode_subtitle_packet ();
188 decode_audio_packet ();
191 av_packet_unref (&_packet);
195 /** @param data pointer to array of pointers to buffers.
196 * Only the first buffer will be used for non-planar data, otherwise there will be one per channel.
198 shared_ptr<AudioBuffers>
199 FFmpegDecoder::deinterleave_audio (shared_ptr<FFmpegAudioStream> stream) const
201 DCPOMATIC_ASSERT (bytes_per_audio_sample (stream));
203 DCPOMATIC_DISABLE_WARNINGS
204 int const size = av_samples_get_buffer_size (
205 0, stream->stream(_format_context)->codec->channels, _frame->nb_samples, audio_sample_format (stream), 1
207 DCPOMATIC_ENABLE_WARNINGS
209 /* XXX: can't we just use _frame->nb_samples directly here? */
210 /* XXX: can't we use swr_convert() to do the format conversion? */
212 /* Deinterleave and convert to float */
214 /* total_samples and frames will be rounded down here, so if there are stray samples at the end
215 of the block that do not form a complete sample or frame they will be dropped.
217 int const total_samples = size / bytes_per_audio_sample (stream);
218 int const channels = stream->channels();
219 int const frames = total_samples / channels;
220 shared_ptr<AudioBuffers> audio (new AudioBuffers (channels, frames));
221 float** data = audio->data();
223 switch (audio_sample_format (stream)) {
224 case AV_SAMPLE_FMT_U8:
226 uint8_t* p = reinterpret_cast<uint8_t *> (_frame->data[0]);
229 for (int i = 0; i < total_samples; ++i) {
230 data[channel][sample] = float(*p++) / (1 << 23);
233 if (channel == channels) {
241 case AV_SAMPLE_FMT_S16:
243 int16_t* p = reinterpret_cast<int16_t *> (_frame->data[0]);
246 for (int i = 0; i < total_samples; ++i) {
247 data[channel][sample] = float(*p++) / (1 << 15);
250 if (channel == channels) {
258 case AV_SAMPLE_FMT_S16P:
260 int16_t** p = reinterpret_cast<int16_t **> (_frame->data);
261 for (int i = 0; i < channels; ++i) {
262 for (int j = 0; j < frames; ++j) {
263 data[i][j] = static_cast<float>(p[i][j]) / (1 << 15);
269 case AV_SAMPLE_FMT_S32:
271 int32_t* p = reinterpret_cast<int32_t *> (_frame->data[0]);
274 for (int i = 0; i < total_samples; ++i) {
275 data[channel][sample] = static_cast<float>(*p++) / 2147483648;
278 if (channel == channels) {
286 case AV_SAMPLE_FMT_S32P:
288 int32_t** p = reinterpret_cast<int32_t **> (_frame->data);
289 for (int i = 0; i < channels; ++i) {
290 for (int j = 0; j < frames; ++j) {
291 data[i][j] = static_cast<float>(p[i][j]) / 2147483648;
297 case AV_SAMPLE_FMT_FLT:
299 float* p = reinterpret_cast<float*> (_frame->data[0]);
302 for (int i = 0; i < total_samples; ++i) {
303 data[channel][sample] = *p++;
306 if (channel == channels) {
314 case AV_SAMPLE_FMT_FLTP:
316 float** p = reinterpret_cast<float**> (_frame->data);
317 DCPOMATIC_ASSERT (_frame->channels <= channels);
318 /* Sometimes there aren't as many channels in the _frame as in the stream */
319 for (int i = 0; i < _frame->channels; ++i) {
320 memcpy (data[i], p[i], frames * sizeof(float));
322 for (int i = _frame->channels; i < channels; ++i) {
323 audio->make_silent (i);
329 throw DecodeError (String::compose (_("Unrecognised audio sample format (%1)"), static_cast<int> (audio_sample_format (stream))));
336 FFmpegDecoder::audio_sample_format (shared_ptr<FFmpegAudioStream> stream) const
338 DCPOMATIC_DISABLE_WARNINGS
339 return stream->stream (_format_context)->codec->sample_fmt;
340 DCPOMATIC_ENABLE_WARNINGS
344 FFmpegDecoder::bytes_per_audio_sample (shared_ptr<FFmpegAudioStream> stream) const
346 return av_get_bytes_per_sample (audio_sample_format (stream));
350 FFmpegDecoder::seek (ContentTime time, bool accurate)
352 Decoder::seek (time, accurate);
354 /* If we are doing an `accurate' seek, we need to use pre-roll, as
355 we don't really know what the seek will give us.
358 ContentTime pre_roll = accurate ? ContentTime::from_seconds (2) : ContentTime (0);
361 /* XXX: it seems debatable whether PTS should be used here...
362 http://www.mjbshaw.com/2012/04/seeking-in-ffmpeg-know-your-timestamp.html
365 optional<int> stream;
368 stream = _video_stream;
370 DCPOMATIC_ASSERT (_ffmpeg_content->audio);
371 shared_ptr<FFmpegAudioStream> s = dynamic_pointer_cast<FFmpegAudioStream> (_ffmpeg_content->audio->stream ());
373 stream = s->index (_format_context);
377 DCPOMATIC_ASSERT (stream);
379 ContentTime u = time - _pts_offset;
380 if (u < ContentTime ()) {
386 u.seconds() / av_q2d (_format_context->streams[stream.get()]->time_base),
391 /* Force re-creation of filter graphs to reset them and hence to make sure
392 they don't have any pre-seek frames knocking about.
394 boost::mutex::scoped_lock lm (_filter_graphs_mutex);
395 _filter_graphs.clear ();
398 if (video_codec_context ()) {
399 avcodec_flush_buffers (video_codec_context());
402 DCPOMATIC_DISABLE_WARNINGS
403 BOOST_FOREACH (shared_ptr<FFmpegAudioStream> i, ffmpeg_content()->ffmpeg_audio_streams()) {
404 avcodec_flush_buffers (i->stream(_format_context)->codec);
406 DCPOMATIC_ENABLE_WARNINGS
408 if (subtitle_codec_context ()) {
409 avcodec_flush_buffers (subtitle_codec_context ());
412 _have_current_subtitle = false;
414 BOOST_FOREACH (optional<ContentTime>& i, _next_time) {
415 i = optional<ContentTime>();
420 FFmpegDecoder::decode_audio_packet ()
422 /* Audio packets can contain multiple frames, so we may have to call avcodec_decode_audio4
426 AVPacket copy_packet = _packet;
427 int const stream_index = copy_packet.stream_index;
429 /* XXX: inefficient */
430 vector<shared_ptr<FFmpegAudioStream> > streams = ffmpeg_content()->ffmpeg_audio_streams ();
431 vector<shared_ptr<FFmpegAudioStream> >::const_iterator stream = streams.begin ();
432 while (stream != streams.end () && !(*stream)->uses_index (_format_context, stream_index)) {
436 if (stream == streams.end ()) {
437 /* The packet's stream may not be an audio one; just ignore it in this method if so */
441 DCPOMATIC_DISABLE_WARNINGS
442 while (copy_packet.size > 0) {
445 int decode_result = avcodec_decode_audio4 ((*stream)->stream (_format_context)->codec, _frame, &frame_finished, ©_packet);
446 if (decode_result < 0) {
447 /* avcodec_decode_audio4 can sometimes return an error even though it has decoded
448 some valid data; for example dca_subframe_footer can return AVERROR_INVALIDDATA
449 if it overreads the auxiliary data. ffplay carries on if frame_finished is true,
450 even in the face of such an error, so I think we should too.
452 Returning from the method here caused mantis #352.
454 LOG_WARNING ("avcodec_decode_audio4 failed (%1)", decode_result);
456 /* Fudge decode_result so that we come out of the while loop when
457 we've processed this data.
459 decode_result = copy_packet.size;
462 if (frame_finished) {
463 shared_ptr<AudioBuffers> data = deinterleave_audio (*stream);
466 if (_frame->pts == AV_NOPTS_VALUE) {
467 /* In some streams we see not every frame coming through with a timestamp; for those
468 that have AV_NOPTS_VALUE we need to work out the timestamp ourselves. This is
469 particularly noticeable with TrueHD streams (see #1111).
471 if (_next_time[stream_index]) {
472 ct = *_next_time[stream_index];
475 ct = ContentTime::from_seconds (
476 av_frame_get_best_effort_timestamp (_frame) *
477 av_q2d ((*stream)->stream (_format_context)->time_base))
481 _next_time[stream_index] = ct + ContentTime::from_frames(data->frames(), (*stream)->frame_rate());
483 if (ct < ContentTime ()) {
484 /* Discard audio data that comes before time 0 */
485 Frame const remove = min (int64_t (data->frames()), (-ct).frames_ceil(double((*stream)->frame_rate ())));
486 data->move (data->frames() - remove, remove, 0);
487 data->set_frames (data->frames() - remove);
488 ct += ContentTime::from_frames (remove, (*stream)->frame_rate ());
491 if (ct < ContentTime()) {
493 "Crazy timestamp %1 for %2 samples in stream %3 packet pts %4 (ts=%5 tb=%6, off=%7)",
496 copy_packet.stream_index,
498 av_frame_get_best_effort_timestamp(_frame),
499 av_q2d((*stream)->stream(_format_context)->time_base),
500 to_string(_pts_offset)
503 DCPOMATIC_ENABLE_WARNINGS
505 /* Give this data provided there is some, and its time is sane */
506 if (ct >= ContentTime() && data->frames() > 0) {
507 audio->emit (film(), *stream, data, ct);
511 copy_packet.data += decode_result;
512 copy_packet.size -= decode_result;
517 FFmpegDecoder::decode_video_packet ()
519 DCPOMATIC_ASSERT (_video_stream);
522 DCPOMATIC_DISABLE_WARNINGS
523 if (avcodec_decode_video2 (video_codec_context(), _frame, &frame_finished, &_packet) < 0 || !frame_finished) {
526 DCPOMATIC_ENABLE_WARNINGS
528 boost::mutex::scoped_lock lm (_filter_graphs_mutex);
530 shared_ptr<VideoFilterGraph> graph;
532 list<shared_ptr<VideoFilterGraph> >::iterator i = _filter_graphs.begin();
533 while (i != _filter_graphs.end() && !(*i)->can_process (dcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format)) {
537 if (i == _filter_graphs.end ()) {
538 dcp::Fraction vfr (lrint(_ffmpeg_content->video_frame_rate().get() * 1000), 1000);
539 graph.reset (new VideoFilterGraph (dcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format, vfr));
540 graph->setup (_ffmpeg_content->filters ());
541 _filter_graphs.push_back (graph);
542 LOG_GENERAL (N_("New graph for %1x%2, pixel format %3"), _frame->width, _frame->height, _frame->format);
547 list<pair<shared_ptr<Image>, int64_t> > images = graph->process (_frame);
549 for (list<pair<shared_ptr<Image>, int64_t> >::iterator i = images.begin(); i != images.end(); ++i) {
551 shared_ptr<Image> image = i->first;
553 if (i->second != AV_NOPTS_VALUE) {
554 double const pts = i->second * av_q2d (_format_context->streams[_video_stream.get()]->time_base) + _pts_offset.seconds ();
558 shared_ptr<ImageProxy> (new RawImageProxy (image)),
559 llrint(pts * _ffmpeg_content->active_video_frame_rate(film()))
562 LOG_WARNING_NC ("Dropping frame without PTS");
570 FFmpegDecoder::decode_subtitle_packet ()
574 if (avcodec_decode_subtitle2 (subtitle_codec_context(), &sub, &got_subtitle, &_packet) < 0 || !got_subtitle) {
578 /* Stop any current subtitle, either at the time it was supposed to stop, or now if now is sooner */
579 if (_have_current_subtitle) {
580 if (_current_subtitle_to) {
581 only_text()->emit_stop (min(*_current_subtitle_to, subtitle_period(sub).from + _pts_offset));
583 only_text()->emit_stop (subtitle_period(sub).from + _pts_offset);
585 _have_current_subtitle = false;
588 if (sub.num_rects <= 0) {
589 /* Nothing new in this subtitle */
593 /* Subtitle PTS (within the source, not taking into account any of the
594 source that we may have chopped off for the DCP).
596 FFmpegSubtitlePeriod sub_period = subtitle_period (sub);
598 from = sub_period.from + _pts_offset;
600 _current_subtitle_to = *sub_period.to + _pts_offset;
602 _current_subtitle_to = optional<ContentTime>();
603 _have_current_subtitle = true;
606 for (unsigned int i = 0; i < sub.num_rects; ++i) {
607 AVSubtitleRect const * rect = sub.rects[i];
609 switch (rect->type) {
612 case SUBTITLE_BITMAP:
613 decode_bitmap_subtitle (rect, from);
616 cout << "XXX: SUBTITLE_TEXT " << rect->text << "\n";
619 decode_ass_subtitle (rect->ass, from);
624 if (_current_subtitle_to) {
625 only_text()->emit_stop (*_current_subtitle_to);
628 avsubtitle_free (&sub);
632 FFmpegDecoder::decode_bitmap_subtitle (AVSubtitleRect const * rect, ContentTime from)
634 /* Note BGRA is expressed little-endian, so the first byte in the word is B, second
635 G, third R, fourth A.
637 shared_ptr<Image> image (new Image (AV_PIX_FMT_BGRA, dcp::Size (rect->w, rect->h), true));
639 #ifdef DCPOMATIC_HAVE_AVSUBTITLERECT_PICT
640 /* Start of the first line in the subtitle */
641 uint8_t* sub_p = rect->pict.data[0];
642 /* sub_p looks up into a BGRA palette which is at rect->pict.data[1];
643 (i.e. first byte B, second G, third R, fourth A)
645 uint8_t const * palette = rect->pict.data[1];
647 /* Start of the first line in the subtitle */
648 uint8_t* sub_p = rect->data[0];
649 /* sub_p looks up into a BGRA palette which is at rect->data[1].
650 (first byte B, second G, third R, fourth A)
652 uint8_t const * palette = rect->data[1];
654 /* And the stream has a map of those palette colours to colours
655 chosen by the user; created a `mapped' palette from those settings.
657 map<RGBA, RGBA> colour_map = ffmpeg_content()->subtitle_stream()->colours ();
658 vector<RGBA> mapped_palette (rect->nb_colors);
659 for (int i = 0; i < rect->nb_colors; ++i) {
660 RGBA c (palette[2], palette[1], palette[0], palette[3]);
661 map<RGBA, RGBA>::const_iterator j = colour_map.find (c);
662 if (j != colour_map.end ()) {
663 mapped_palette[i] = j->second;
665 /* This colour was not found in the FFmpegSubtitleStream's colour map; probably because
666 it is from a project that was created before this stuff was added. Just use the
667 colour straight from the original palette.
669 mapped_palette[i] = c;
674 /* Start of the output data */
675 uint8_t* out_p = image->data()[0];
677 for (int y = 0; y < rect->h; ++y) {
678 uint8_t* sub_line_p = sub_p;
679 uint8_t* out_line_p = out_p;
680 for (int x = 0; x < rect->w; ++x) {
681 RGBA const p = mapped_palette[*sub_line_p++];
687 #ifdef DCPOMATIC_HAVE_AVSUBTITLERECT_PICT
688 sub_p += rect->pict.linesize[0];
690 sub_p += rect->linesize[0];
692 out_p += image->stride()[0];
695 int target_width = subtitle_codec_context()->width;
696 if (target_width == 0 && video_codec_context()) {
697 /* subtitle_codec_context()->width == 0 has been seen in the wild but I don't
698 know if it's supposed to mean something from FFmpeg's point of view.
700 target_width = video_codec_context()->width;
702 int target_height = subtitle_codec_context()->height;
703 if (target_height == 0 && video_codec_context()) {
704 target_height = video_codec_context()->height;
706 DCPOMATIC_ASSERT (target_width);
707 DCPOMATIC_ASSERT (target_height);
708 dcpomatic::Rect<double> const scaled_rect (
709 static_cast<double> (rect->x) / target_width,
710 static_cast<double> (rect->y) / target_height,
711 static_cast<double> (rect->w) / target_width,
712 static_cast<double> (rect->h) / target_height
715 only_text()->emit_bitmap_start (from, image, scaled_rect);
719 FFmpegDecoder::decode_ass_subtitle (string ass, ContentTime from)
721 /* We have no styles and no Format: line, so I'm assuming that FFmpeg
722 produces a single format of Dialogue: lines...
727 for (size_t i = 0; i < ass.length(); ++i) {
728 if (commas < 9 && ass[i] == ',') {
730 } else if (commas == 9) {
739 sub::RawSubtitle base;
740 list<sub::RawSubtitle> raw = sub::SSAReader::parse_line (
743 _ffmpeg_content->video->size().width,
744 _ffmpeg_content->video->size().height
747 BOOST_FOREACH (sub::Subtitle const & i, sub::collect<list<sub::Subtitle> > (raw)) {
748 only_text()->emit_plain_start (from, i);