From: Carl Hetherington Date: Wed, 26 Jun 2013 00:21:21 +0000 (+0100) Subject: Hacks. X-Git-Tag: v2.0.48~1337^2~304^2~5 X-Git-Url: https://git.carlh.net/gitweb/?p=dcpomatic.git;a=commitdiff_plain;h=d0d584a7dde6de383302615634fdee17e9724fe8 Hacks. --- diff --git a/doc/design/timing.tex b/doc/design/timing.tex new file mode 100644 index 000000000..567ba024f --- /dev/null +++ b/doc/design/timing.tex @@ -0,0 +1,42 @@ +\documentclass{article} +\begin{document} + +We are trying to implement full-ish playlist based content specification. The timing is awkward. + +\section{Reference timing} + +Frame rates of things can vary a lot; content can be in pretty much +anything, and DCP video and audio frame rates may change on a whim +depending on what is best for a given set of content. This suggests +(albeit without strong justification) the need for a frame-rate-independent unit of time. + +So far we've been using a time type called \texttt{Time} expressed in +$\mathtt{TIME\_HZ}^{-1}$; e.g. \texttt{TIME\_HZ} units is 1 second. +\texttt{TIME\_HZ} is chosen to be divisible by lots of frame and +sample rates. + +We express content start time as a \texttt{Time}. + + +\section{Timing at different stages of the chain} + +Let's try this: decoders produce sequences of (perhaps) video frames +and (perhaps) audio frames. There are no gaps. They are at the +content's native frame rates and are synchronised (meaning that if +they are played together, at the content's frame rates, they will be +in sync). The decoders give timestamps for each piece of their +output, which are \emph{simple indices} (\texttt{ContentVideoFrame} +and \texttt{ContentAudioFrame}). Decoders know nothing of \texttt{Time}. + + +\section{Split of stuff between decoders and player} + +In some ways it seems nice to have decoders which produce the rawest +possible data and make the player sort it out (e.g.\ cropping and +scaling video, resampling audio). The resampling is awkward, though, +as you really need one resampler per source. So it might make more sense +to put stuff in the decoder. But then, what's one map of resamplers between friends? + + + +\end{document} diff --git a/src/lib/audio_content.h b/src/lib/audio_content.h index 73a00ca7d..9bf53e0ab 100644 --- a/src/lib/audio_content.h +++ b/src/lib/audio_content.h @@ -41,6 +41,8 @@ public: class AudioContent : public virtual Content { public: + typedef int64_t Frame; + AudioContent (boost::shared_ptr, Time); AudioContent (boost::shared_ptr, boost::filesystem::path); AudioContent (boost::shared_ptr, boost::shared_ptr); @@ -49,7 +51,7 @@ public: void as_xml (xmlpp::Node *) const; virtual int audio_channels () const = 0; - virtual ContentAudioFrame audio_length () const = 0; + virtual AudioContent::Frame audio_length () const = 0; virtual int content_audio_frame_rate () const = 0; virtual int output_audio_frame_rate () const = 0; virtual AudioMapping audio_mapping () const = 0; diff --git a/src/lib/audio_decoder.cc b/src/lib/audio_decoder.cc index a9e01908c..396471910 100644 --- a/src/lib/audio_decoder.cc +++ b/src/lib/audio_decoder.cc @@ -31,57 +31,12 @@ using std::cout; using boost::optional; using boost::shared_ptr; -AudioDecoder::AudioDecoder (shared_ptr f, shared_ptr c) +AudioDecoder::AudioDecoder (shared_ptr f) : Decoder (f) - , _next_audio (0) - , _audio_content (c) + , _next_audio_frame (0) { - if (_audio_content->content_audio_frame_rate() != _audio_content->output_audio_frame_rate()) { - - shared_ptr film = _film.lock (); - assert (film); - - stringstream s; - s << String::compose ( - "Will resample audio from %1 to %2", - _audio_content->content_audio_frame_rate(), _audio_content->output_audio_frame_rate() - ); - - film->log()->log (s.str ()); - - /* We will be using planar float data when we call the - resampler. As far as I can see, the audio channel - layout is not necessary for our purposes; it seems - only to be used get the number of channels and - decide if rematrixing is needed. It won't be, since - input and output layouts are the same. - */ - - _swr_context = swr_alloc_set_opts ( - 0, - av_get_default_channel_layout (_audio_content->audio_channels ()), - AV_SAMPLE_FMT_FLTP, - _audio_content->output_audio_frame_rate(), - av_get_default_channel_layout (_audio_content->audio_channels ()), - AV_SAMPLE_FMT_FLTP, - _audio_content->content_audio_frame_rate(), - 0, 0 - ); - - swr_init (_swr_context); - } else { - _swr_context = 0; - } } -AudioDecoder::~AudioDecoder () -{ - if (_swr_context) { - swr_free (&_swr_context); - } -} - - #if 0 void AudioDecoder::process_end () @@ -113,54 +68,8 @@ AudioDecoder::process_end () #endif void -AudioDecoder::audio (shared_ptr data, Time time) -{ - /* Maybe resample */ - if (_swr_context) { - - /* Compute the resampled frames count and add 32 for luck */ - int const max_resampled_frames = ceil ( - (int64_t) data->frames() * _audio_content->output_audio_frame_rate() / _audio_content->content_audio_frame_rate() - ) + 32; - - shared_ptr resampled (new AudioBuffers (data->channels(), max_resampled_frames)); - - /* Resample audio */ - int const resampled_frames = swr_convert ( - _swr_context, (uint8_t **) resampled->data(), max_resampled_frames, (uint8_t const **) data->data(), data->frames() - ); - - if (resampled_frames < 0) { - throw EncodeError (_("could not run sample-rate converter")); - } - - resampled->set_frames (resampled_frames); - - /* And point our variables at the resampled audio */ - data = resampled; - } - - shared_ptr film = _film.lock (); - assert (film); - - /* Remap channels */ - shared_ptr dcp_mapped (new AudioBuffers (film->dcp_audio_channels(), data->frames())); - dcp_mapped->make_silent (); - list > map = _audio_content->audio_mapping().content_to_dcp (); - for (list >::iterator i = map.begin(); i != map.end(); ++i) { - dcp_mapped->accumulate_channel (data.get(), i->first, i->second); - } - - Audio (dcp_mapped, time); - _next_audio = time + film->audio_frames_to_time (data->frames()); -} - -bool -AudioDecoder::audio_done () const +AudioDecoder::audio (shared_ptr data, AudioContent::Frame frame) { - shared_ptr film = _film.lock (); - assert (film); - - return (_audio_content->length() - _next_audio) < film->audio_frames_to_time (1); + Audio (data, frame); + _next_audio_frame = frame + data->frames (); } - diff --git a/src/lib/audio_decoder.h b/src/lib/audio_decoder.h index 1da8a676f..168348c2e 100644 --- a/src/lib/audio_decoder.h +++ b/src/lib/audio_decoder.h @@ -24,33 +24,26 @@ #ifndef DCPOMATIC_AUDIO_DECODER_H #define DCPOMATIC_AUDIO_DECODER_H -#include "audio_source.h" #include "decoder.h" -extern "C" { -#include -} +#include "content.h" -class AudioContent; +class AudioBuffers; /** @class AudioDecoder. * @brief Parent class for audio decoders. */ -class AudioDecoder : public AudioSource, public virtual Decoder +class AudioDecoder : public virtual Decoder { public: - AudioDecoder (boost::shared_ptr, boost::shared_ptr); - ~AudioDecoder (); + AudioDecoder (boost::shared_ptr); -protected: - - void audio (boost::shared_ptr, Time); - bool audio_done () const; + /** Emitted when some audio data is ready */ + boost::signals2::signal, AudioContent::Frame)> Audio; - Time _next_audio; - boost::shared_ptr _audio_content; +protected: -private: - SwrContext* _swr_context; + void audio (boost::shared_ptr, AudioContent::Frame); + AudioContent::Frame _next_audio_frame; }; #endif diff --git a/src/lib/audio_sink.h b/src/lib/audio_sink.h deleted file mode 100644 index 1aad5edf9..000000000 --- a/src/lib/audio_sink.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -#ifndef DCPOMATIC_AUDIO_SINK_H -#define DCPOMATIC_AUDIO_SINK_H - -class AudioBuffers; - -class AudioSink -{ -public: - /** Call with some audio data */ - virtual void process_audio (boost::shared_ptr, Time) = 0; -}; - -#endif diff --git a/src/lib/audio_source.cc b/src/lib/audio_source.cc deleted file mode 100644 index e61721646..000000000 --- a/src/lib/audio_source.cc +++ /dev/null @@ -1,42 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -#include "audio_source.h" -#include "audio_sink.h" - -using boost::shared_ptr; -using boost::weak_ptr; -using boost::bind; - -static void -process_audio_proxy (weak_ptr sink, shared_ptr audio, Time time) -{ - shared_ptr p = sink.lock (); - if (p) { - p->process_audio (audio, time); - } -} - -void -AudioSource::connect_audio (shared_ptr s) -{ - Audio.connect (bind (process_audio_proxy, weak_ptr (s), _1, _2)); -} - - diff --git a/src/lib/audio_source.h b/src/lib/audio_source.h deleted file mode 100644 index ef47e969b..000000000 --- a/src/lib/audio_source.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -/** @file src/audio_source.h - * @brief Parent class for classes which emit audio data. - */ - -#ifndef DCPOMATIC_AUDIO_SOURCE_H -#define DCPOMATIC_AUDIO_SOURCE_H - -#include -#include "types.h" - -class AudioBuffers; -class AudioSink; - -/** A class that emits audio data */ -class AudioSource -{ -public: - /** Emitted when some audio data is ready */ - boost::signals2::signal, Time)> Audio; - - void connect_audio (boost::shared_ptr); -}; - -#endif diff --git a/src/lib/black_decoder.cc b/src/lib/black_decoder.cc index 0b231edd3..beb6bfef3 100644 --- a/src/lib/black_decoder.cc +++ b/src/lib/black_decoder.cc @@ -25,7 +25,8 @@ using boost::shared_ptr; BlackDecoder::BlackDecoder (shared_ptr f, shared_ptr c) : Decoder (f) - , VideoDecoder (f, c) + , VideoDecoder (f) + , _null_content (c) { } @@ -36,9 +37,9 @@ BlackDecoder::pass () if (!_image) { _image.reset (new SimpleImage (AV_PIX_FMT_RGB24, video_size(), true)); _image->make_black (); - video (_image, false, _next_video); + video (_image, false, _next_video_frame); } else { - video (_image, true, _next_video); + video (_image, true, _next_video_frame); } } @@ -53,50 +54,28 @@ BlackDecoder::video_frame_rate () const return f->dcp_video_frame_rate (); } -ContentVideoFrame +VideoContent::Frame BlackDecoder::video_length () const { - return _video_content->length() * video_frame_rate() / TIME_HZ; -} - -Time -BlackDecoder::position () const -{ - return _next_video; + return _null_content->length() * video_frame_rate() / TIME_HZ; } void -BlackDecoder::seek (Time t) +BlackDecoder::seek (VideoContent::Frame frame) { - _next_video = t; + _next_video_frame = frame; } void BlackDecoder::seek_back () { - boost::shared_ptr f = _film.lock (); - if (!f) { - return; - } - - _next_video -= f->video_frames_to_time (2); -} - -void -BlackDecoder::seek_forward () -{ - boost::shared_ptr f = _film.lock (); - if (!f) { - return; + if (_next_video_frame > 0) { + --_next_video_frame; } - - _next_video += f->video_frames_to_time (1); } - + bool BlackDecoder::done () const { - return video_done (); + return _next_video_frame >= _null_content->video_length (); } - - diff --git a/src/lib/black_decoder.h b/src/lib/black_decoder.h index 4591881a1..40aa73d72 100644 --- a/src/lib/black_decoder.h +++ b/src/lib/black_decoder.h @@ -29,20 +29,19 @@ public: /* Decoder */ void pass (); - void seek (Time); - void seek_back (); - void seek_forward (); - Time position () const; bool done () const; /* VideoDecoder */ + void seek (VideoContent::Frame); + void seek_back (); float video_frame_rate () const; libdcp::Size video_size () const { return libdcp::Size (256, 256); } - ContentVideoFrame video_length () const; + VideoContent::Frame video_length () const; private: + boost::shared_ptr _null_content; boost::shared_ptr _image; }; diff --git a/src/lib/combiner.cc b/src/lib/combiner.cc deleted file mode 100644 index 44971d135..000000000 --- a/src/lib/combiner.cc +++ /dev/null @@ -1,65 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -#include "combiner.h" -#include "image.h" - -using boost::shared_ptr; - -Combiner::Combiner () -{ - -} - -/** Process video for the left half of the frame. - * Subtitle parameter will be ignored. - * @param image Frame image. - */ -void -Combiner::process_video (shared_ptr image, bool, Time) -{ - _image.reset (new SimpleImage (image, true)); -} - -/** Process video for the right half of the frame. - * @param image Frame image. - * @param sub Subtitle (which will be put onto the whole frame) - */ -void -Combiner::process_video_b (shared_ptr image, bool, Time t) -{ - /* Copy the right half of this image into our _image */ - /* XXX: this should probably be in the Image class */ - for (int i = 0; i < image->components(); ++i) { - int const line_size = image->line_size()[i]; - int const half_line_size = line_size / 2; - - uint8_t* p = _image->data()[i]; - uint8_t* q = image->data()[i]; - - for (int j = 0; j < image->lines (i); ++j) { - memcpy (p + half_line_size, q + half_line_size, half_line_size); - p += _image->stride()[i]; - q += image->stride()[i]; - } - } - - Video (_image, false, t); - _image.reset (); -} diff --git a/src/lib/combiner.h b/src/lib/combiner.h deleted file mode 100644 index 46c90b4d8..000000000 --- a/src/lib/combiner.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -/** @file src/lib/combiner.h - * @brief Class for combining two video streams. - */ - -#include "video_source.h" -#include "video_sink.h" - -/** @class Combiner - * @brief A class which can combine two video streams into one, with - * one image used for the left half of the screen and the other for - * the right. - */ -class Combiner : public VideoSource, public VideoSink -{ -public: - Combiner (); - - void process_video (boost::shared_ptr i, bool, Time); - void process_video_b (boost::shared_ptr i, bool, Time); - -private: - /** The image that we are currently working on */ - boost::shared_ptr _image; -}; diff --git a/src/lib/decoder.h b/src/lib/decoder.h index 391b9d19a..cfca6867f 100644 --- a/src/lib/decoder.h +++ b/src/lib/decoder.h @@ -29,8 +29,6 @@ #include #include #include -#include "video_source.h" -#include "audio_source.h" #include "film.h" class Image; @@ -54,24 +52,6 @@ public: */ virtual void pass () = 0; - /** Seek this decoder to as close as possible to some time, - * expressed relative to our source's start. - * @param t Time. - * @param a true to try hard to be accurate, otherwise false. - */ - virtual void seek (Time) = 0; - - /** Seek back one video frame */ - virtual void seek_back () = 0; - - /** Seek forward one video frame */ - virtual void seek_forward () = 0; - - /** @return Approximate time of the next content that we will emit, - * expressed relative to the start of our source. - */ - virtual Time position () const = 0; - virtual bool done () const = 0; protected: diff --git a/src/lib/encoder.cc b/src/lib/encoder.cc index 8b2db0eb3..c3865d2c1 100644 --- a/src/lib/encoder.cc +++ b/src/lib/encoder.cc @@ -169,7 +169,7 @@ Encoder::frame_done () } void -Encoder::process_video (shared_ptr image, bool same, Time) +Encoder::process_video (shared_ptr image, bool same) { boost::mutex::scoped_lock lock (_mutex); @@ -215,7 +215,7 @@ Encoder::process_video (shared_ptr image, bool same, Time) } void -Encoder::process_audio (shared_ptr data, Time) +Encoder::process_audio (shared_ptr data) { _writer->write (data); } diff --git a/src/lib/encoder.h b/src/lib/encoder.h index 3fe707b51..b5a641f50 100644 --- a/src/lib/encoder.h +++ b/src/lib/encoder.h @@ -36,8 +36,6 @@ extern "C" { #include } #include "util.h" -#include "video_sink.h" -#include "audio_sink.h" class Image; class AudioBuffers; @@ -55,7 +53,7 @@ class Job; * is supplied as uncompressed PCM in blocks of various sizes. */ -class Encoder : public VideoSink, public AudioSink +class Encoder { public: Encoder (boost::shared_ptr f, boost::shared_ptr); @@ -68,10 +66,10 @@ public: * @param i Video frame image. * @param same true if i is the same as the last time we were called. */ - void process_video (boost::shared_ptr i, bool same, Time); + void process_video (boost::shared_ptr i, bool same); /** Call with some audio data */ - void process_audio (boost::shared_ptr, Time); + void process_audio (boost::shared_ptr); /** Called when a processing run has finished */ void process_end (); diff --git a/src/lib/ffmpeg_content.cc b/src/lib/ffmpeg_content.cc index 68132c5ab..378bd98cb 100644 --- a/src/lib/ffmpeg_content.cc +++ b/src/lib/ffmpeg_content.cc @@ -139,7 +139,7 @@ FFmpegContent::examine (shared_ptr job) shared_ptr examiner (new FFmpegExaminer (shared_from_this ())); - ContentVideoFrame video_length = 0; + VideoContent::Frame video_length = 0; video_length = examiner->video_length (); film->log()->log (String::compose ("Video length obtained from header as %1 frames", video_length)); @@ -214,12 +214,12 @@ FFmpegContent::set_audio_stream (shared_ptr s) signal_changed (FFmpegContentProperty::AUDIO_STREAM); } -ContentAudioFrame +AudioContent::Frame FFmpegContent::audio_length () const { int const cafr = content_audio_frame_rate (); int const vfr = video_frame_rate (); - ContentVideoFrame const vl = video_length (); + VideoContent::Frame const vl = video_length (); boost::mutex::scoped_lock lm (_mutex); if (!_audio_stream) { diff --git a/src/lib/ffmpeg_content.h b/src/lib/ffmpeg_content.h index 36c24c2b3..fc45267ee 100644 --- a/src/lib/ffmpeg_content.h +++ b/src/lib/ffmpeg_content.h @@ -99,7 +99,7 @@ public: /* AudioContent */ int audio_channels () const; - ContentAudioFrame audio_length () const; + AudioContent::Frame audio_length () const; int content_audio_frame_rate () const; int output_audio_frame_rate () const; AudioMapping audio_mapping () const; diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc index f1d984ee1..d897aef9d 100644 --- a/src/lib/ffmpeg_decoder.cc +++ b/src/lib/ffmpeg_decoder.cc @@ -59,8 +59,8 @@ using libdcp::Size; FFmpegDecoder::FFmpegDecoder (shared_ptr f, shared_ptr c, bool video, bool audio) : Decoder (f) - , VideoDecoder (f, c) - , AudioDecoder (f, c) + , VideoDecoder (f) + , AudioDecoder (f) , FFmpeg (c) , _subtitle_codec_context (0) , _subtitle_codec (0) @@ -108,7 +108,8 @@ FFmpegDecoder::pass () } /* Stop us being asked for any more data */ - _next_video = _next_audio = _ffmpeg_content->length (); + _next_video_frame = _ffmpeg_content->video_length (); + _next_audio_frame = _ffmpeg_content->audio_length (); return; } @@ -119,6 +120,7 @@ FFmpegDecoder::pass () } else if (_ffmpeg_content->audio_stream() && _packet.stream_index == _ffmpeg_content->audio_stream()->id && _decode_audio) { decode_audio_packet (); } else if (_ffmpeg_content->subtitle_stream() && _packet.stream_index == _ffmpeg_content->subtitle_stream()->id) { +#if 0 int got_subtitle; AVSubtitle sub; @@ -138,6 +140,7 @@ FFmpegDecoder::pass () } avsubtitle_free (&sub); } +#endif } av_free_packet (&_packet); @@ -256,38 +259,26 @@ FFmpegDecoder::bytes_per_audio_sample () const } void -FFmpegDecoder::seek (Time t) +FFmpegDecoder::seek (VideoContent::Frame frame) { - do_seek (t, false, false); - VideoDecoder::seek (t); + do_seek (frame, false, false); } void FFmpegDecoder::seek_back () { - if (position() < (2.5 * TIME_HZ / _ffmpeg_content->video_frame_rate())) { + if (_next_video_frame == 0) { return; } - do_seek (position() - 2.5 * TIME_HZ / _ffmpeg_content->video_frame_rate(), true, true); + do_seek (_next_video_frame - 1, true, true); VideoDecoder::seek_back (); } void -FFmpegDecoder::seek_forward () +FFmpegDecoder::do_seek (VideoContent::Frame frame, bool backwards, bool accurate) { - if (position() >= (_ffmpeg_content->length() - 0.5 * TIME_HZ / _ffmpeg_content->video_frame_rate())) { - return; - } - - do_seek (position() - 0.5 * TIME_HZ / _ffmpeg_content->video_frame_rate(), true, true); - VideoDecoder::seek_forward (); -} - -void -FFmpegDecoder::do_seek (Time t, bool backwards, bool accurate) -{ - int64_t const vt = t / (av_q2d (_format_context->streams[_video_stream]->time_base) * TIME_HZ); + int64_t const vt = frame * _ffmpeg_content->video_frame_rate() / av_q2d (_format_context->streams[_video_stream]->time_base); av_seek_frame (_format_context, _video_stream, vt, backwards ? AVSEEK_FLAG_BACKWARD : 0); avcodec_flush_buffers (video_codec_context()); @@ -347,7 +338,7 @@ FFmpegDecoder::decode_audio_packet () ); assert (audio_codec_context()->channels == _ffmpeg_content->audio_channels()); - audio (deinterleave_audio (_frame->data, data_size), source_pts_seconds * TIME_HZ); + Audio (deinterleave_audio (_frame->data, data_size), source_pts_seconds * _ffmpeg_content->content_audio_frame_rate()); } copy_packet.data += decode_result; @@ -398,11 +389,33 @@ FFmpegDecoder::decode_video_packet () int64_t const bet = av_frame_get_best_effort_timestamp (_frame); if (bet != AV_NOPTS_VALUE) { - /* XXX: may need to insert extra frames / remove frames here ... - (as per old Matcher) - */ - Time const t = bet * av_q2d (_format_context->streams[_video_stream]->time_base) * TIME_HZ; - video (image, false, t); + + double const pts = bet * av_q2d (_format_context->streams[_video_stream]->time_base); + double const next = _next_video_frame / _ffmpeg_content->video_frame_rate(); + double const one_frame = 1 / _ffmpeg_content->video_frame_rate (); + double delta = pts - next; + + while (delta > one_frame) { + /* This PTS is more than one frame forward in time of where we think we should be; emit + a black frame. + */ + boost::shared_ptr black ( + new SimpleImage ( + static_cast (_frame->format), + libdcp::Size (video_codec_context()->width, video_codec_context()->height), + true + ) + ); + + black->make_black (); + video (image, false, _next_video_frame); + delta -= one_frame; + } + + if (delta > -one_frame) { + /* This PTS is within a frame of being right; emit this (otherwise it will be dropped) */ + video (image, false, _next_video_frame); + } } else { shared_ptr film = _film.lock (); assert (film); @@ -413,27 +426,6 @@ FFmpegDecoder::decode_video_packet () return true; } -Time -FFmpegDecoder::position () const -{ - if (_decode_video && _decode_audio && _ffmpeg_content->audio_stream()) { - return min (_next_video, _next_audio); - } - - if (_decode_audio && _ffmpeg_content->audio_stream()) { - return _next_audio; - } - - return _next_video; -} - -bool -FFmpegDecoder::done () const -{ - bool const ad = !_decode_audio || !_ffmpeg_content->audio_stream() || audio_done(); - bool const vd = !_decode_video || video_done(); - return ad && vd; -} void FFmpegDecoder::setup_subtitle () @@ -455,3 +447,12 @@ FFmpegDecoder::setup_subtitle () throw DecodeError (N_("could not open subtitle decoder")); } } + +bool +FFmpegDecoder::done () const +{ + bool const vd = !_decode_video || (_next_video_frame >= _ffmpeg_content->video_length()); + bool const ad = !_decode_audio || !_ffmpeg_content->audio_stream() || (_next_audio_frame >= _ffmpeg_content->audio_length()); + return vd && ad; +} + diff --git a/src/lib/ffmpeg_decoder.h b/src/lib/ffmpeg_decoder.h index 331d9be70..a8eabb972 100644 --- a/src/lib/ffmpeg_decoder.h +++ b/src/lib/ffmpeg_decoder.h @@ -49,10 +49,8 @@ public: ~FFmpegDecoder (); void pass (); - void seek (Time); + void seek (VideoContent::Frame); void seek_back (); - void seek_forward (); - Time position () const; bool done () const; private: @@ -65,7 +63,7 @@ private: AVSampleFormat audio_sample_format () const; int bytes_per_audio_sample () const; - void do_seek (Time, bool, bool); + void do_seek (VideoContent::Frame, bool, bool); bool decode_video_packet (); void decode_audio_packet (); diff --git a/src/lib/ffmpeg_examiner.cc b/src/lib/ffmpeg_examiner.cc index c09395e76..6f1524f50 100644 --- a/src/lib/ffmpeg_examiner.cc +++ b/src/lib/ffmpeg_examiner.cc @@ -134,7 +134,7 @@ FFmpegExaminer::video_size () const } /** @return Length (in video frames) according to our content's header */ -ContentVideoFrame +VideoContent::Frame FFmpegExaminer::video_length () const { return (double (_format_context->duration) / AV_TIME_BASE) * video_frame_rate(); diff --git a/src/lib/ffmpeg_examiner.h b/src/lib/ffmpeg_examiner.h index 57b7775d4..5cf9c2d0a 100644 --- a/src/lib/ffmpeg_examiner.h +++ b/src/lib/ffmpeg_examiner.h @@ -31,7 +31,7 @@ public: float video_frame_rate () const; libdcp::Size video_size () const; - ContentVideoFrame video_length () const; + VideoContent::Frame video_length () const; std::vector > subtitle_streams () const { return _subtitle_streams; diff --git a/src/lib/image.cc b/src/lib/image.cc index 0bff1a7cc..722ff5d3c 100644 --- a/src/lib/image.cc +++ b/src/lib/image.cc @@ -43,8 +43,9 @@ extern "C" { #include "i18n.h" -using namespace std; -using namespace boost; +using std::string; +using std::min; +using boost::shared_ptr; using libdcp::Size; void diff --git a/src/lib/imagemagick_content.cc b/src/lib/imagemagick_content.cc index f9ee8cc84..2fd65ffa0 100644 --- a/src/lib/imagemagick_content.cc +++ b/src/lib/imagemagick_content.cc @@ -87,7 +87,7 @@ ImageMagickContent::clone () const } void -ImageMagickContent::set_video_length (ContentVideoFrame len) +ImageMagickContent::set_video_length (VideoContent::Frame len) { { boost::mutex::scoped_lock lm (_mutex); diff --git a/src/lib/imagemagick_content.h b/src/lib/imagemagick_content.h index d7673d870..04425af08 100644 --- a/src/lib/imagemagick_content.h +++ b/src/lib/imagemagick_content.h @@ -43,7 +43,7 @@ public: boost::shared_ptr clone () const; Time length () const; - void set_video_length (ContentVideoFrame); + void set_video_length (VideoContent::Frame); static bool valid_file (boost::filesystem::path); }; diff --git a/src/lib/imagemagick_decoder.cc b/src/lib/imagemagick_decoder.cc index c9123c77c..acc34421c 100644 --- a/src/lib/imagemagick_decoder.cc +++ b/src/lib/imagemagick_decoder.cc @@ -34,7 +34,7 @@ using libdcp::Size; ImageMagickDecoder::ImageMagickDecoder (shared_ptr f, shared_ptr c) : Decoder (f) - , VideoDecoder (f, c) + , VideoDecoder (f) , ImageMagick (c) { @@ -43,12 +43,12 @@ ImageMagickDecoder::ImageMagickDecoder (shared_ptr f, shared_ptr= _imagemagick_content->length ()) { + if (_next_video_frame >= _imagemagick_content->video_length ()) { return; } if (_image) { - video (_image, true, _next_video); + video (_image, true, _next_video_frame); return; } @@ -71,48 +71,25 @@ ImageMagickDecoder::pass () delete magick_image; - _image = _image->crop (_imagemagick_content->crop(), true); - video (_image, false, _next_video); + video (_image, false, _next_video_frame); } void -ImageMagickDecoder::seek (Time t) +ImageMagickDecoder::seek (VideoContent::Frame frame) { - _next_video = t; + _next_video_frame = frame; } void ImageMagickDecoder::seek_back () { - boost::shared_ptr f = _film.lock (); - if (!f) { - return; - } - - _next_video -= f->video_frames_to_time (2); -} - -void -ImageMagickDecoder::seek_forward () -{ - boost::shared_ptr f = _film.lock (); - if (!f) { - return; + if (_next_video_frame > 0) { + _next_video_frame--; } - - _next_video += f->video_frames_to_time (1); } -Time -ImageMagickDecoder::position () const -{ - return _next_video; -} - - bool ImageMagickDecoder::done () const { - return video_done (); + return _next_video_frame > _imagemagick_content->video_length (); } - diff --git a/src/lib/imagemagick_decoder.h b/src/lib/imagemagick_decoder.h index e169f9bc3..286f47337 100644 --- a/src/lib/imagemagick_decoder.h +++ b/src/lib/imagemagick_decoder.h @@ -34,10 +34,8 @@ public: /* Decoder */ void pass (); - void seek (Time); + void seek (VideoContent::Frame); void seek_back (); - void seek_forward (); - Time position () const; bool done () const; private: diff --git a/src/lib/imagemagick_examiner.h b/src/lib/imagemagick_examiner.h index 827dad67e..801ede442 100644 --- a/src/lib/imagemagick_examiner.h +++ b/src/lib/imagemagick_examiner.h @@ -33,7 +33,7 @@ public: float video_frame_rate () const; libdcp::Size video_size () const; - ContentVideoFrame video_length () const; + VideoContent::Frame video_length () const; private: boost::weak_ptr _film; diff --git a/src/lib/matcher.cc b/src/lib/matcher.cc deleted file mode 100644 index 4acb82afa..000000000 --- a/src/lib/matcher.cc +++ /dev/null @@ -1,224 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -#include "matcher.h" -#include "image.h" -#include "log.h" - -#include "i18n.h" - -using std::min; -using std::cout; -using std::list; -using boost::shared_ptr; - -Matcher::Matcher (shared_ptr log, int sample_rate, float frames_per_second) - : Processor (log) - , _sample_rate (sample_rate) - , _frames_per_second (frames_per_second) - , _video_frames (0) - , _audio_frames (0) - , _had_first_video (false) - , _had_first_audio (false) -{ - -} - -void -Matcher::process_video (boost::shared_ptr image, bool same, boost::shared_ptr sub, double t) -{ - _pixel_format = image->pixel_format (); - _size = image->size (); - - _log->log(String::compose("Matcher video @ %1 [audio=%2, video=%3, pending_audio=%4]", t, _audio_frames, _video_frames, _pending_audio.size())); - - if (!_first_input || t < _first_input.get()) { - _first_input = t; - } - - bool const this_is_first_video = !_had_first_video; - _had_first_video = true; - - if (!_had_first_audio) { - /* No audio yet; we must postpone these data until we have some */ - _pending_video.push_back (VideoRecord (image, same, sub, t)); - } else if (this_is_first_video && _had_first_audio) { - /* First video since we got audio */ - _pending_video.push_back (VideoRecord (image, same, sub, t)); - fix_start (); - } else { - /* Normal running */ - - /* Difference between where this video is and where it should be */ - double const delta = t - _first_input.get() - _video_frames / _frames_per_second; - double const one_frame = 1 / _frames_per_second; - - if (delta > one_frame) { - /* Insert frames to make up the difference */ - int const extra = rint (delta / one_frame); - for (int i = 0; i < extra; ++i) { - repeat_last_video (); - _log->log (String::compose ("Extra video frame inserted at %1s", _video_frames / _frames_per_second)); - } - } - - if (delta > -one_frame) { - Video (image, same, sub); - ++_video_frames; - } else { - /* We are omitting a frame to keep things right */ - _log->log (String::compose ("Frame removed at %1s; delta %2; first input was at %3", t, delta, _first_input.get())); - } - - _last_image = image; - _last_subtitle = sub; - } -} - -void -Matcher::process_audio (boost::shared_ptr b, double t) -{ - _channels = b->channels (); - - _log->log (String::compose ( - "Matcher audio (%1 frames) @ %2 [video=%3, audio=%4, pending_video=%5, pending_audio=%6]", - b->frames(), t, _video_frames, _audio_frames, _pending_video.size(), _pending_audio.size() - ) - ); - - if (!_first_input || t < _first_input.get()) { - _first_input = t; - } - - bool const this_is_first_audio = !_had_first_audio; - _had_first_audio = true; - - if (!_had_first_video) { - /* No video yet; we must postpone these data until we have some */ - _pending_audio.push_back (AudioRecord (b, t)); - } else if (this_is_first_audio && _had_first_video) { - /* First audio since we got video */ - _pending_audio.push_back (AudioRecord (b, t)); - fix_start (); - } else { - /* Normal running. We assume audio time stamps are consecutive, so there's no equivalent of - the checking / insertion of repeat frames that there is for video. - */ - Audio (b); - _audio_frames += b->frames (); - } -} - -void -Matcher::process_end () -{ - if (_audio_frames == 0 || !_pixel_format || !_size || !_channels) { - /* We won't do anything */ - return; - } - - _log->log (String::compose ("Matcher has seen %1 video frames (which equals %2 audio frames) and %3 audio frames", - _video_frames, video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second), _audio_frames)); - - match ((double (_audio_frames) / _sample_rate) - (double (_video_frames) / _frames_per_second)); -} - -void -Matcher::fix_start () -{ - assert (!_pending_video.empty ()); - assert (!_pending_audio.empty ()); - - _log->log (String::compose ("Fixing start; video at %1, audio at %2", _pending_video.front().time, _pending_audio.front().time)); - - match (_pending_video.front().time - _pending_audio.front().time); - - for (list::iterator i = _pending_video.begin(); i != _pending_video.end(); ++i) { - process_video (i->image, i->same, i->subtitle, i->time); - } - - _pending_video.clear (); - - for (list::iterator i = _pending_audio.begin(); i != _pending_audio.end(); ++i) { - process_audio (i->audio, i->time); - } - - _pending_audio.clear (); -} - -void -Matcher::match (double extra_video_needed) -{ - _log->log (String::compose ("Match %1", extra_video_needed)); - - if (extra_video_needed > 0) { - - /* Emit black video frames */ - - int const black_video_frames = ceil (extra_video_needed * _frames_per_second); - - _log->log (String::compose (N_("Emitting %1 frames of black video"), black_video_frames)); - - shared_ptr black (new SimpleImage (_pixel_format.get(), _size.get(), true)); - black->make_black (); - for (int i = 0; i < black_video_frames; ++i) { - Video (black, i != 0, shared_ptr()); - ++_video_frames; - } - - extra_video_needed -= black_video_frames / _frames_per_second; - } - - if (extra_video_needed < 0) { - - /* Emit silence */ - - int64_t to_do = -extra_video_needed * _sample_rate; - _log->log (String::compose (N_("Emitting %1 frames of silence"), to_do)); - - /* Do things in half second blocks as I think there may be limits - to what FFmpeg (and in particular the resampler) can cope with. - */ - int64_t const block = _sample_rate / 2; - shared_ptr b (new AudioBuffers (_channels.get(), block)); - b->make_silent (); - - while (to_do > 0) { - int64_t const this_time = min (to_do, block); - b->set_frames (this_time); - Audio (b); - _audio_frames += b->frames (); - to_do -= this_time; - } - } -} - -void -Matcher::repeat_last_video () -{ - if (!_last_image) { - shared_ptr im (new SimpleImage (_pixel_format.get(), _size.get(), true)); - im->make_black (); - _last_image = im; - } - - Video (_last_image, true, _last_subtitle); - ++_video_frames; -} - diff --git a/src/lib/matcher.h b/src/lib/matcher.h deleted file mode 100644 index 61fd81436..000000000 --- a/src/lib/matcher.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - Copyright (C) 2012 Carl Hetherington - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - -*/ - -#include -#include "processor.h" - -class Matcher : public Processor, public TimedAudioSink, public TimedVideoSink, public AudioSource, public VideoSource -{ -public: - Matcher (boost::shared_ptr log, int sample_rate, float frames_per_second); - void process_video (boost::shared_ptr i, bool, boost::shared_ptr s, double); - void process_audio (boost::shared_ptr, double); - void process_end (); - -private: - void fix_start (); - void match (double); - void repeat_last_video (); - - int _sample_rate; - float _frames_per_second; - int _video_frames; - int64_t _audio_frames; - boost::optional _pixel_format; - boost::optional _size; - boost::optional _channels; - - struct VideoRecord { - VideoRecord (boost::shared_ptr i, bool s, boost::shared_ptr u, double t) - : image (i) - , same (s) - , subtitle (u) - , time (t) - {} - - boost::shared_ptr image; - bool same; - boost::shared_ptr subtitle; - double time; - }; - - struct AudioRecord { - AudioRecord (boost::shared_ptr a, double t) - : audio (a) - , time (t) - {} - - boost::shared_ptr audio; - double time; - }; - - std::list _pending_video; - std::list _pending_audio; - - boost::optional _first_input; - boost::shared_ptr _last_image; - boost::shared_ptr _last_subtitle; - - bool _had_first_video; - bool _had_first_audio; -}; diff --git a/src/lib/null_content.h b/src/lib/null_content.h index 889ff7a0d..44bfffa49 100644 --- a/src/lib/null_content.h +++ b/src/lib/null_content.h @@ -43,7 +43,7 @@ public: int audio_channels () const; - ContentAudioFrame audio_length () const { + AudioContent::Frame audio_length () const { return _audio_length; } @@ -62,6 +62,6 @@ public: } private: - ContentAudioFrame _audio_length; + AudioContent::Frame _audio_length; Time _length; }; diff --git a/src/lib/player.cc b/src/lib/player.cc index cd1c54d5b..79f1c3b97 100644 --- a/src/lib/player.cc +++ b/src/lib/player.cc @@ -32,12 +32,16 @@ #include "null_content.h" #include "black_decoder.h" #include "silence_decoder.h" +#include "ratio.h" +#include "resampler.h" using std::list; using std::cout; using std::min; using std::max; using std::vector; +using std::pair; +using std::map; using boost::shared_ptr; using boost::weak_ptr; using boost::dynamic_pointer_cast; @@ -49,10 +53,12 @@ struct Piece Piece (shared_ptr c, shared_ptr d) : content (c) , decoder (d) + , last_emission (0) {} shared_ptr content; shared_ptr decoder; + Time last_emission; }; @@ -125,13 +131,8 @@ Player::pass () continue; } - if (!_audio && dynamic_pointer_cast ((*i)->decoder) && !dynamic_pointer_cast ((*i)->decoder)) { - continue; - } - - Time const t = (*i)->content->start() + (*i)->decoder->position(); - if (t < earliest_t) { - earliest_t = t; + if ((*i)->last_emission < earliest_t) { + earliest_t = (*i)->last_emission; earliest = *i; } } @@ -142,39 +143,106 @@ Player::pass () } earliest->decoder->pass (); - _position = earliest->content->start() + earliest->decoder->position (); + _position = earliest->last_emission; return false; } void -Player::process_video (weak_ptr weak_content, shared_ptr image, bool same, Time time) +Player::process_video (weak_ptr weak_piece, shared_ptr image, bool same, VideoContent::Frame frame) { - shared_ptr content = weak_content.lock (); - if (!content) { + shared_ptr piece = weak_piece.lock (); + if (!piece) { + return; + } + + shared_ptr content = dynamic_pointer_cast (piece->content); + assert (content); + + FrameRateConversion frc (content->video_frame_rate(), _film->dcp_video_frame_rate()); + if (frc.skip && (frame % 2) == 1) { return; } + + image = image->crop (content->crop(), true); + + libdcp::Size const container_size = _video_container_size.get_value_or (_film->container()->size (_film->full_frame ())); + libdcp::Size const image_size = content->ratio()->size (container_size); - time += content->start (); + image = image->scale_and_convert_to_rgb (image_size, _film->scaler(), true); + +#if 0 + if (film->with_subtitles ()) { + shared_ptr sub; + if (_timed_subtitle && _timed_subtitle->displayed_at (t)) { + sub = _timed_subtitle->subtitle (); + } + + if (sub) { + dcpomatic::Rect const tx = subtitle_transformed_area ( + float (image_size.width) / content->video_size().width, + float (image_size.height) / content->video_size().height, + sub->area(), film->subtitle_offset(), film->subtitle_scale() + ); + + shared_ptr im = sub->image()->scale (tx.size(), film->scaler(), true); + image->alpha_blend (im, tx.position()); + } + } +#endif + + if (image_size != container_size) { + assert (image_size.width <= container_size.width); + assert (image_size.height <= container_size.height); + shared_ptr im (new SimpleImage (PIX_FMT_RGB24, container_size, true)); + im->make_black (); + im->copy (image, Position ((container_size.width - image_size.width) / 2, (container_size.height - image_size.height) / 2)); + image = im; + } + + Time time = content->start() + (frame * frc.factor() * TIME_HZ / _film->dcp_video_frame_rate()); Video (image, same, time); + + if (frc.repeat) { + time += TIME_HZ / _film->dcp_video_frame_rate(); + Video (image, true, time); + } + + piece->last_emission = min (piece->last_emission, time); } void -Player::process_audio (weak_ptr weak_content, shared_ptr audio, Time time) +Player::process_audio (weak_ptr weak_piece, shared_ptr audio, AudioContent::Frame frame) { - shared_ptr content = weak_content.lock (); - if (!content) { + shared_ptr piece = weak_piece.lock (); + if (!piece) { return; } - + + shared_ptr content = dynamic_pointer_cast (piece->content); + assert (content); + + if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) { + audio = resampler(content)->run (audio); + } + + /* Remap channels */ + shared_ptr dcp_mapped (new AudioBuffers (_film->dcp_audio_channels(), audio->frames())); + dcp_mapped->make_silent (); + list > map = content->audio_mapping().content_to_dcp (); + for (list >::iterator i = map.begin(); i != map.end(); ++i) { + dcp_mapped->accumulate_channel (audio.get(), i->first, i->second); + } + /* The time of this audio may indicate that some of our buffered audio is not going to be added to any more, so it can be emitted. */ - time += content->start (); + Time const time = content->start() + (frame * TIME_HZ / _film->dcp_audio_frame_rate()); + piece->last_emission = min (piece->last_emission, time); - cout << "Player gets " << audio->frames() << " @ " << time << " cf " << _next_audio << "\n"; + cout << "Player gets " << dcp_mapped->frames() << " @ " << time << " cf " << _next_audio << "\n"; if (time > _next_audio) { /* We can emit some audio from our buffers */ @@ -224,10 +292,18 @@ Player::seek (Time t) } for (list >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) { - Time s = t - (*i)->content->start (); + shared_ptr vc = dynamic_pointer_cast ((*i)->content); + if (!vc) { + continue; + } + + Time s = t - vc->start (); s = max (static_cast