From 9c399a21b37d83ceb2c81706975e2c46d1a3f673 Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Wed, 18 Dec 2013 09:39:36 +0000 Subject: [PATCH] Considerable rework of decoder timing; tests pass, at least. --- src/lib/audio_decoder.cc | 12 +- src/lib/audio_decoder.h | 11 +- src/lib/decoder.cc | 29 +++ src/lib/decoder.h | 24 ++- src/lib/ffmpeg_decoder.cc | 93 ++++---- src/lib/ffmpeg_decoder.h | 11 +- src/lib/image_decoder.cc | 27 +-- src/lib/image_decoder.h | 9 +- src/lib/player.cc | 414 +++++++++++++++--------------------- src/lib/player.h | 44 ++-- src/lib/resampler.cc | 8 +- src/lib/resampler.h | 2 +- src/lib/sndfile_decoder.cc | 18 +- src/lib/sndfile_decoder.h | 9 +- src/lib/subtitle_decoder.cc | 3 +- src/lib/subtitle_decoder.h | 3 +- src/lib/video_decoder.cc | 9 +- src/lib/video_decoder.h | 13 +- src/wx/film_editor.cc | 2 +- test/frame_rate_test.cc | 2 +- test/play_test.cc | 6 +- test/resampler_test.cc | 7 +- 22 files changed, 347 insertions(+), 409 deletions(-) diff --git a/src/lib/audio_decoder.cc b/src/lib/audio_decoder.cc index a73ad4d7c..7ff8529c6 100644 --- a/src/lib/audio_decoder.cc +++ b/src/lib/audio_decoder.cc @@ -42,15 +42,5 @@ AudioDecoder::AudioDecoder (shared_ptr film, shared_ptr data, ContentTime time) { - Audio (data, time); -} - -/** This is a bit odd, but necessary when we have (e.g.) FFmpegDecoders with no audio. - * The player needs to know that there is no audio otherwise it will keep trying to - * pass() the decoder to get it to emit audio. - */ -bool -AudioDecoder::has_audio () const -{ - return _audio_content->audio_channels () > 0; + _pending.push_back (shared_ptr (new DecodedAudio (data, time))); } diff --git a/src/lib/audio_decoder.h b/src/lib/audio_decoder.h index 10e4298f7..0cd0e9754 100644 --- a/src/lib/audio_decoder.h +++ b/src/lib/audio_decoder.h @@ -27,6 +27,7 @@ #include "decoder.h" #include "content.h" #include "audio_content.h" +#include "decoded.h" class AudioBuffers; @@ -38,15 +39,15 @@ class AudioDecoder : public virtual Decoder public: AudioDecoder (boost::shared_ptr, boost::shared_ptr); - bool has_audio () const; - - /** Emitted when some audio data is ready */ - boost::signals2::signal, ContentTime)> Audio; + boost::shared_ptr audio_content () const { + return _audio_content; + } protected: void audio (boost::shared_ptr, ContentTime); - boost::shared_ptr _audio_content; + + boost::shared_ptr _audio_content; }; #endif diff --git a/src/lib/decoder.cc b/src/lib/decoder.cc index 3f4cda6eb..18c5acd35 100644 --- a/src/lib/decoder.cc +++ b/src/lib/decoder.cc @@ -36,3 +36,32 @@ Decoder::Decoder (shared_ptr f) { } + +shared_ptr +Decoder::peek () +{ + while (_pending.empty () && !pass ()) {} + + if (_pending.empty ()) { + return shared_ptr (); + } + + return _pending.front (); +} + +shared_ptr +Decoder::get () +{ + shared_ptr d = peek (); + if (d) { + _pending.pop_front (); + } + + return d; +} + +void +Decoder::seek (ContentTime, bool) +{ + _pending.clear (); +} diff --git a/src/lib/decoder.h b/src/lib/decoder.h index aa36d41b4..654cacad4 100644 --- a/src/lib/decoder.h +++ b/src/lib/decoder.h @@ -30,6 +30,7 @@ #include "types.h" class Film; +class Decoded; /** @class Decoder. * @brief Parent class for decoders of content. @@ -40,26 +41,31 @@ public: Decoder (boost::shared_ptr); virtual ~Decoder () {} - /** Perform one decode pass of the content, which may or may not - * cause the object to emit some data. - */ - virtual void pass () = 0; - - /** Seek so that the next pass() will yield the next thing + /** Seek so that the next get_*() will yield the next thing * (video/sound frame, subtitle etc.) at or after the requested * time. Pass accurate = true to try harder to get close to * the request. */ - virtual void seek (DCPTime time, bool accurate) = 0; - - virtual bool done () const = 0; + virtual void seek (ContentTime time, bool accurate); + + boost::shared_ptr peek (); + boost::shared_ptr get (); protected: + /** Perform one decode pass of the content, which may or may not + * result in a complete quantum (Decoded object) of decoded stuff + * being made ready. + * @return true if the decoder is done (i.e. no more data will be + * produced by any future calls to pass() without a seek() first). + */ + virtual bool pass () = 0; virtual void flush () {}; /** The Film that we are decoding in */ boost::weak_ptr _film; + + std::list > _pending; }; #endif diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc index d72fed0bc..b2e8cc44a 100644 --- a/src/lib/ffmpeg_decoder.cc +++ b/src/lib/ffmpeg_decoder.cc @@ -88,7 +88,7 @@ FFmpegDecoder::FFmpegDecoder (shared_ptr f, shared_ptrfirst_video(); - bool const have_audio = audio && c->audio_stream() && c->audio_stream()->first_audio; + bool const have_audio = _decode_audio && c->audio_stream () && c->audio_stream()->first_audio; /* First, make one of them start at 0 */ @@ -142,12 +142,15 @@ FFmpegDecoder::flush () decode_audio_packet (); } +#if 0 + /* XXX */ /* Stop us being asked for any more data */ _video_position = _ffmpeg_content->video_length (); _audio_position = _ffmpeg_content->audio_length (); +#endif } -void +bool FFmpegDecoder::pass () { int r = av_read_frame (_format_context, &_packet); @@ -163,7 +166,7 @@ FFmpegDecoder::pass () } flush (); - return; + return true; } avcodec_get_frame_defaults (_frame); @@ -182,6 +185,7 @@ FFmpegDecoder::pass () } av_free_packet (&_packet); + return false; } /** @param data pointer to array of pointers to buffers. @@ -297,14 +301,22 @@ FFmpegDecoder::bytes_per_audio_sample () const } int -FFmpegDecoder::minimal_run (boost::function finished) +FFmpegDecoder::minimal_run (boost::function finished) { int frames_read = 0; - - while (!finished (frames_read)) { + ContentTime last_video = 0; + ContentTime last_audio = 0; + bool flushing = false; + + while (!finished (last_video, last_audio, frames_read)) { int r = av_read_frame (_format_context, &_packet); if (r < 0) { - return -1; + /* We should flush our decoders here, possibly yielding a few more frames, + but the consequence of having to do that is too hideous to contemplate. + Instead we give up and say that you can't seek too close to the end + of a file. + */ + return frames_read; } ++frames_read; @@ -318,31 +330,28 @@ FFmpegDecoder::minimal_run (boost::function finished) int finished = 0; r = avcodec_decode_video2 (video_codec_context(), _frame, &finished, &_packet); if (r >= 0 && finished) { - _video_position = rint ( - (av_frame_get_best_effort_timestamp (_frame) * time_base + _video_pts_offset) * _ffmpeg_content->video_frame_rate() + last_video = rint ( + (av_frame_get_best_effort_timestamp (_frame) * time_base + _video_pts_offset) * TIME_HZ ); } - + } else if (_ffmpeg_content->audio_stream() && _packet.stream_index == _ffmpeg_content->audio_stream()->index (_format_context)) { - AVPacket copy_packet = _packet; - while (copy_packet.size > 0) { int finished; - r = avcodec_decode_audio4 (audio_codec_context(), _frame, &finished, ©_packet); + r = avcodec_decode_audio4 (audio_codec_context(), _frame, &finished, &_packet); if (r >= 0 && finished) { - _audio_position = rint ( - (av_frame_get_best_effort_timestamp (_frame) * time_base + _audio_pts_offset) * - _ffmpeg_content->audio_stream()->frame_rate + last_audio = rint ( + (av_frame_get_best_effort_timestamp (_frame) * time_base + _audio_pts_offset) * TIME_HZ ); } - + copy_packet.data += r; copy_packet.size -= r; } } - + av_free_packet (&_packet); } @@ -350,12 +359,9 @@ FFmpegDecoder::minimal_run (boost::function finished) } bool -FFmpegDecoder::seek_overrun_finished (DCPTime seek) const +FFmpegDecoder::seek_overrun_finished (ContentTime seek, ContentTime last_video, ContentTime last_audio) const { - return ( - _video_position >= _ffmpeg_content->time_to_content_video_frames (seek) || - _audio_position >= _ffmpeg_content->time_to_content_audio_frames (seek, _ffmpeg_content->position()) - ); + return last_video >= seek || last_audio >= seek; } bool @@ -365,16 +371,16 @@ FFmpegDecoder::seek_final_finished (int n, int done) const } void -FFmpegDecoder::seek_and_flush (DCPTime t) +FFmpegDecoder::seek_and_flush (ContentTime t) { - int64_t const initial_v = ((_ffmpeg_content->time_to_content_video_frames (t) / _ffmpeg_content->video_frame_rate()) - _video_pts_offset) / + int64_t const initial_v = ((double (t) / TIME_HZ) - _video_pts_offset) / av_q2d (_format_context->streams[_video_stream]->time_base); av_seek_frame (_format_context, _video_stream, initial_v, AVSEEK_FLAG_BACKWARD); shared_ptr as = _ffmpeg_content->audio_stream (); if (as) { - int64_t initial_a = ((_ffmpeg_content->time_to_content_audio_frames (t, t) / as->frame_rate) - _audio_pts_offset) / + int64_t initial_a = ((double (t) / TIME_HZ) - _audio_pts_offset) / av_q2d (as->stream(_format_context)->time_base); av_seek_frame (_format_context, as->index (_format_context), initial_a, AVSEEK_FLAG_BACKWARD); @@ -387,21 +393,20 @@ FFmpegDecoder::seek_and_flush (DCPTime t) if (_subtitle_codec_context) { avcodec_flush_buffers (_subtitle_codec_context); } - - _video_position = _ffmpeg_content->time_to_content_video_frames (t); - _audio_position = _ffmpeg_content->time_to_content_audio_frames (t, t); } void -FFmpegDecoder::seek (DCPTime time, bool accurate) +FFmpegDecoder::seek (ContentTime time, bool accurate) { + Decoder::seek (time, accurate); + /* If we are doing an accurate seek, our initial shot will be 200ms (200 being a number plucked from the air) earlier than we want to end up. The loop below will hopefully then step through to where we want to be. */ - DCPTime pre_roll = accurate ? (0.2 * TIME_HZ) : 0; - DCPTime initial_seek = time - pre_roll; + ContentTime pre_roll = accurate ? (0.2 * TIME_HZ) : 0; + ContentTime initial_seek = time - pre_roll; if (initial_seek < 0) { initial_seek = 0; } @@ -415,11 +420,11 @@ FFmpegDecoder::seek (DCPTime time, bool accurate) return; } - int const N = minimal_run (boost::bind (&FFmpegDecoder::seek_overrun_finished, this, time)); - + int const N = minimal_run (boost::bind (&FFmpegDecoder::seek_overrun_finished, this, time, _1, _2)); + seek_and_flush (initial_seek); if (N > 0) { - minimal_run (boost::bind (&FFmpegDecoder::seek_final_finished, this, N - 1, _1)); + minimal_run (boost::bind (&FFmpegDecoder::seek_final_finished, this, N - 1, _3)); } } @@ -445,9 +450,11 @@ FFmpegDecoder::decode_audio_packet () } if (frame_finished) { - Time const t = (av_q2d (_format_context->streams[copy_packet.stream_index]->time_base) - * av_frame_get_best_effort_timestamp(_frame) + _audio_pts_offset) * TIME_HZ; - + ContentTime const t = rint ( + (av_q2d (_format_context->streams[copy_packet.stream_index]->time_base) + * av_frame_get_best_effort_timestamp(_frame) + _audio_pts_offset) * TIME_HZ + ); + int const data_size = av_samples_get_buffer_size ( 0, audio_codec_context()->channels, _frame->nb_samples, audio_sample_format (), 1 ); @@ -501,7 +508,7 @@ FFmpegDecoder::decode_video_packet () } if (i->second != AV_NOPTS_VALUE) { - Time const t = (i->second * av_q2d (_format_context->streams[_video_stream]->time_base) + _video_pts_offset) * TIME_HZ; + ContentTime const t = rint ((i->second * av_q2d (_format_context->streams[_video_stream]->time_base) + _video_pts_offset) * TIME_HZ); video (image, false, t); } else { shared_ptr film = _film.lock (); @@ -535,14 +542,6 @@ FFmpegDecoder::setup_subtitle () } } -bool -FFmpegDecoder::done () const -{ - bool const vd = !_decode_video || (_video_position >= _ffmpeg_content->video_length()); - bool const ad = !_decode_audio || !_ffmpeg_content->audio_stream() || (_audio_position >= _ffmpeg_content->audio_length()); - return vd && ad; -} - void FFmpegDecoder::decode_subtitle_packet () { diff --git a/src/lib/ffmpeg_decoder.h b/src/lib/ffmpeg_decoder.h index ed4557702..e50b248cf 100644 --- a/src/lib/ffmpeg_decoder.h +++ b/src/lib/ffmpeg_decoder.h @@ -51,15 +51,12 @@ public: FFmpegDecoder (boost::shared_ptr, boost::shared_ptr, bool video, bool audio); ~FFmpegDecoder (); - void pass (); - void seek (DCPTime time, bool); - bool done () const; + void seek (ContentTime time, bool); private: friend class ::ffmpeg_pts_offset_test; - static double compute_pts_offset (double, double, float); - + bool pass (); void flush (); void setup_subtitle (); @@ -74,9 +71,9 @@ private: void maybe_add_subtitle (); boost::shared_ptr deinterleave_audio (uint8_t** data, int size); - bool seek_overrun_finished (DCPTime) const; + bool seek_overrun_finished (ContentTime, ContentTime, ContentTime) const; bool seek_final_finished (int, int) const; - int minimal_run (boost::function); + int minimal_run (boost::function); void seek_and_flush (int64_t); AVCodecContext* _subtitle_codec_context; ///< may be 0 if there is no subtitle diff --git a/src/lib/image_decoder.cc b/src/lib/image_decoder.cc index 723690247..9e90b5bc8 100644 --- a/src/lib/image_decoder.cc +++ b/src/lib/image_decoder.cc @@ -36,20 +36,22 @@ ImageDecoder::ImageDecoder (shared_ptr f, shared_ptr= _image_content->video_length ()) { - return; + return true; } if (_image && _image_content->still ()) { - video (_image, true, _video_position); - return; + video (_image, true, _video_position * TIME_HZ / _video_content->video_frame_rate ()); + ++_video_position; + return false; } Magick::Image* magick_image = new Magick::Image (_image_content->path (_image_content->still() ? 0 : _video_position).string ()); @@ -73,17 +75,16 @@ ImageDecoder::pass () delete magick_image; - video (_image, false, _video_position); -} + video (_image, false, _video_position * TIME_HZ / _video_content->video_frame_rate ()); + ++_video_position; -void -ImageDecoder::seek (DCPTime time, bool) -{ - _video_position = _video_content->time_to_content_video_frames (time); + return false; } -bool -ImageDecoder::done () const +void +ImageDecoder::seek (ContentTime time, bool accurate) { - return _video_position >= _image_content->video_length (); + Decoder::seek (time, accurate); + + _video_position = rint (time * _video_content->video_frame_rate() / TIME_HZ); } diff --git a/src/lib/image_decoder.h b/src/lib/image_decoder.h index 346fffdf7..f4d81dfb5 100644 --- a/src/lib/image_decoder.h +++ b/src/lib/image_decoder.h @@ -34,14 +34,13 @@ public: return _image_content; } - /* Decoder */ - - void pass (); - void seek (DCPTime, bool); - bool done () const; + void seek (ContentTime, bool); private: + bool pass (); + boost::shared_ptr _image_content; boost::shared_ptr _image; + ContentTime _video_position; }; diff --git a/src/lib/player.cc b/src/lib/player.cc index 7f5e78681..56bf0767d 100644 --- a/src/lib/player.cc +++ b/src/lib/player.cc @@ -45,79 +45,20 @@ using std::map; using boost::shared_ptr; using boost::weak_ptr; using boost::dynamic_pointer_cast; +using boost::optional; class Piece { public: - Piece (shared_ptr c) - : content (c) - , video_position (c->position ()) - , audio_position (c->position ()) - , repeat_to_do (0) - , repeat_done (0) - {} - - Piece (shared_ptr c, shared_ptr d) + Piece (shared_ptr c, shared_ptr d, FrameRateChange f) : content (c) , decoder (d) - , video_position (c->position ()) - , audio_position (c->position ()) + , frc (f) {} - /** Set this piece to repeat a video frame a given number of times */ - void set_repeat (IncomingVideo video, int num) - { - repeat_video = video; - repeat_to_do = num; - repeat_done = 0; - } - - void reset_repeat () - { - repeat_video.image.reset (); - repeat_to_do = 0; - repeat_done = 0; - } - - bool repeating () const - { - return repeat_done != repeat_to_do; - } - - void repeat (Player* player) - { - shared_ptr p = repeat_video.weak_piece.lock (); - if (!p) { - return; - } - - shared_ptr vc = dynamic_pointer_cast (p->content); - if (!vc) { - return; - } - - player->process_video ( - repeat_video.weak_piece, - repeat_video.image, - repeat_video.eyes, - repeat_done > 0, - repeat_video.time, - (repeat_done + 1) * (TIME_HZ / vc->video_frame_rate ()) - ); - - ++repeat_done; - } - shared_ptr content; shared_ptr decoder; - /** DCPTime of the last video we emitted relative to the start of the DCP */ - DCPTime video_position; - /** DCPTime of the last audio we emitted relative to the start of the DCP */ - DCPTime audio_position; - - IncomingVideo repeat_video; - int repeat_to_do; - int repeat_done; + FrameRateChange frc; }; Player::Player (shared_ptr f, shared_ptr p) @@ -130,6 +71,7 @@ Player::Player (shared_ptr f, shared_ptr p) , _audio_position (0) , _audio_merger (f->audio_channels(), bind (&Film::time_to_audio_frames, f.get(), _1), bind (&Film::audio_frames_to_time, f.get(), _1)) , _last_emit_was_black (false) + , _just_did_inaccurate_seek (false) { _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this)); _playlist_content_changed_connection = _playlist->ContentChanged.connect (bind (&Player::content_changed, this, _1, _2, _3)); @@ -156,110 +98,97 @@ Player::pass () setup_pieces (); } - DCPTime earliest_t = TIME_MAX; - shared_ptr earliest; - enum { - VIDEO, - AUDIO - } type = VIDEO; + /* Interrogate all our pieces to find the one with the earliest decoded data */ + + shared_ptr earliest_piece; + shared_ptr earliest_decoded; + DCPTime earliest_time = TIME_MAX; + DCPTime earliest_audio = TIME_MAX; for (list >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) { - if ((*i)->decoder->done ()) { - continue; - } - shared_ptr vd = dynamic_pointer_cast ((*i)->decoder); - shared_ptr ad = dynamic_pointer_cast ((*i)->decoder); + shared_ptr dec = (*i)->decoder->peek (); - if (_video && vd) { - if ((*i)->video_position < earliest_t) { - earliest_t = (*i)->video_position; - earliest = *i; - type = VIDEO; - } + if (dec) { + dec->set_dcp_times ((*i)->frc.speed_up, (*i)->content->position()); } - if (_audio && ad && ad->has_audio ()) { - if ((*i)->audio_position < earliest_t) { - earliest_t = (*i)->audio_position; - earliest = *i; - type = AUDIO; + /* XXX: don't know what to do with this */ +#if 0 + if (ad->done()) { + shared_ptr ac = dynamic_pointer_cast ((*i)->content); + assert (ac); + shared_ptr re = resampler (ac, false); + if (re) { + shared_ptr b = re->flush (); + if (b->frames ()) { + process_audio (earliest, b, ac->audio_length ()); + } } } - } +#endif - if (!earliest) { + if (dec && dec->dcp_time < earliest_time) { + earliest_piece = *i; + earliest_decoded = dec; + earliest_time = dec->dcp_time; + } + + if (dynamic_pointer_cast (dec) && dec->dcp_time < earliest_audio) { + earliest_audio = dec->dcp_time; + } + } + + if (!earliest_piece) { flush (); return true; } - switch (type) { - case VIDEO: - if (earliest_t > _video_position) { + if (earliest_audio != TIME_MAX) { + TimedAudioBuffers tb = _audio_merger.pull (earliest_audio); + Audio (tb.audio, tb.time); + _audio_position += _film->audio_frames_to_time (tb.audio->frames ()); + } + + /* Emit the earliest thing */ + + shared_ptr dv = dynamic_pointer_cast (earliest_decoded); + shared_ptr da = dynamic_pointer_cast (earliest_decoded); + shared_ptr ds = dynamic_pointer_cast (earliest_decoded); + + if (dv) { + if (!_just_did_inaccurate_seek && earliest_time > _video_position) { + /* XXX: if we're inside some content, repeat the last frame... otherwise emit black */ emit_black (); } else { - if (earliest->repeating ()) { - earliest->repeat (this); - } else { - earliest->decoder->pass (); - } + emit_video (earliest_piece, dv); + earliest_piece->decoder->get (); } - break; - - case AUDIO: - if (earliest_t > _audio_position) { - emit_silence (_film->time_to_audio_frames (earliest_t - _audio_position)); + } else if (da) { + if (!_just_did_inaccurate_seek && earliest_time > _audio_position) { + emit_silence (earliest_time - _audio_position); } else { - earliest->decoder->pass (); - - if (earliest->decoder->done()) { - shared_ptr ac = dynamic_pointer_cast (earliest->content); - assert (ac); - shared_ptr re = resampler (ac, false); - if (re) { - shared_ptr b = re->flush (); - if (b->frames ()) { - process_audio (earliest, b, ac->audio_length ()); - } - } - } + emit_audio (earliest_piece, da); + earliest_piece->decoder->get (); } - break; + } else if (ds) { + _in_subtitle.piece = earliest_piece; + _in_subtitle.subtitle = ds; + update_subtitle (); + earliest_piece->decoder->get (); } - if (_audio) { - boost::optional audio_done_up_to; - for (list >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) { - if ((*i)->decoder->done ()) { - continue; - } - - if (dynamic_pointer_cast ((*i)->decoder)) { - audio_done_up_to = min (audio_done_up_to.get_value_or (TIME_MAX), (*i)->audio_position); - } - } - - if (audio_done_up_to) { - TimedAudioBuffers tb = _audio_merger.pull (audio_done_up_to.get ()); - Audio (tb.audio, tb.time); - _audio_position += _film->audio_frames_to_time (tb.audio->frames ()); - } - } + _just_did_inaccurate_seek = false; return false; } -/** @param extra Amount of extra time to add to the content frame's time (for repeat) */ void -Player::process_video (weak_ptr weak_piece, shared_ptr image, Eyes eyes, bool same, ContentTime time, ContentTime extra) +Player::emit_video (weak_ptr weak_piece, shared_ptr video) { /* Keep a note of what came in so that we can repeat it if required */ _last_incoming_video.weak_piece = weak_piece; - _last_incoming_video.image = image; - _last_incoming_video.eyes = eyes; - _last_incoming_video.same = same; - _last_incoming_video.time = time; - _last_incoming_video.extra = extra; + _last_incoming_video.video = video; shared_ptr piece = weak_piece.lock (); if (!piece) { @@ -270,22 +199,23 @@ Player::process_video (weak_ptr weak_piece, shared_ptr image assert (content); FrameRateChange frc (content->video_frame_rate(), _film->video_frame_rate()); +#if 0 + XXX if (frc.skip && (frame % 2) == 1) { return; } +#endif - DCPTime const relative_time = (frame * frc.factor() * TIME_HZ / _film->video_frame_rate()); - if (content->trimmed (relative_time)) { + if (content->trimmed (video->dcp_time - content->position ())) { return; } - DCPTime const time = content->position() + relative_time + extra - content->trim_start (); float const ratio = content->ratio() ? content->ratio()->ratio() : content->video_size_after_crop().ratio(); libdcp::Size const image_size = fit_ratio_within (ratio, _video_container_size); shared_ptr pi ( new PlayerImage ( - image, + video->image, content->crop(), image_size, _video_container_size, @@ -293,32 +223,32 @@ Player::process_video (weak_ptr weak_piece, shared_ptr image ) ); - if (_film->with_subtitles () && _out_subtitle.image && time >= _out_subtitle.from && time <= _out_subtitle.to) { + if ( + _film->with_subtitles () && + _out_subtitle.subtitle->image && + video->dcp_time >= _out_subtitle.subtitle->dcp_time && video->dcp_time <= _out_subtitle.subtitle->dcp_time_to + ) { Position const container_offset ( (_video_container_size.width - image_size.width) / 2, (_video_container_size.height - image_size.width) / 2 ); - pi->set_subtitle (_out_subtitle.image, _out_subtitle.position + container_offset); + pi->set_subtitle (_out_subtitle.subtitle->image, _out_subtitle.position + container_offset); } #ifdef DCPOMATIC_DEBUG _last_video = piece->content; #endif - Video (pi, eyes, content->colour_conversion(), same, time); + Video (pi, video->eyes, content->colour_conversion(), video->same, video->dcp_time); _last_emit_was_black = false; - _video_position = piece->video_position = (time + TIME_HZ / _film->video_frame_rate()); - - if (frc.repeat > 1 && !piece->repeating ()) { - piece->set_repeat (_last_incoming_video, frc.repeat - 1); - } + _video_position = rint (video->dcp_time + TIME_HZ / _film->video_frame_rate()); } void -Player::process_audio (weak_ptr weak_piece, shared_ptr audio, AudioContent::Frame frame) +Player::emit_audio (weak_ptr weak_piece, shared_ptr audio) { shared_ptr piece = weak_piece.lock (); if (!piece) { @@ -330,55 +260,48 @@ Player::process_audio (weak_ptr weak_piece, shared_ptraudio_gain() != 0) { - shared_ptr gain (new AudioBuffers (audio)); + shared_ptr gain (new AudioBuffers (audio->data)); gain->apply_gain (content->audio_gain ()); - audio = gain; + audio->data = gain; } /* Resample */ if (content->content_audio_frame_rate() != content->output_audio_frame_rate()) { - shared_ptr r = resampler (content, true); - pair, AudioContent::Frame> ro = r->run (audio, frame); - audio = ro.first; - frame = ro.second; + audio->data = resampler(content, true)->run (audio->data); } - DCPTime const relative_time = _film->audio_frames_to_time (frame); - - if (content->trimmed (relative_time)) { + if (content->trimmed (audio->dcp_time - content->position ())) { return; } - DCPTime time = content->position() + (content->audio_delay() * TIME_HZ / 1000) + relative_time - content->trim_start (); - /* Remap channels */ - shared_ptr dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames())); + shared_ptr dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->data->frames())); dcp_mapped->make_silent (); list > map = content->audio_mapping().content_to_dcp (); for (list >::iterator i = map.begin(); i != map.end(); ++i) { - if (i->first < audio->channels() && i->second < dcp_mapped->channels()) { - dcp_mapped->accumulate_channel (audio.get(), i->first, i->second); + if (i->first < audio->data->channels() && i->second < dcp_mapped->channels()) { + dcp_mapped->accumulate_channel (audio->data.get(), i->first, i->second); } } - audio = dcp_mapped; + audio->data = dcp_mapped; - /* We must cut off anything that comes before the start of all time */ - if (time < 0) { - int const frames = - time * _film->audio_frame_rate() / TIME_HZ; - if (frames >= audio->frames ()) { + /* Delay */ + audio->dcp_time += content->audio_delay() * TIME_HZ / 1000; + if (audio->dcp_time < 0) { + int const frames = - audio->dcp_time * _film->audio_frame_rate() / TIME_HZ; + if (frames >= audio->data->frames ()) { return; } - shared_ptr trimmed (new AudioBuffers (audio->channels(), audio->frames() - frames)); - trimmed->copy_from (audio.get(), audio->frames() - frames, frames, 0); + shared_ptr trimmed (new AudioBuffers (audio->data->channels(), audio->data->frames() - frames)); + trimmed->copy_from (audio->data.get(), audio->data->frames() - frames, frames, 0); - audio = trimmed; - time = 0; + audio->data = trimmed; + audio->dcp_time = 0; } - _audio_merger.push (audio, time); - piece->audio_position += _film->audio_frames_to_time (audio->frames ()); + _audio_merger.push (audio->data, audio->dcp_time); } void @@ -395,7 +318,7 @@ Player::flush () } while (_audio_position < _video_position) { - emit_silence (_film->time_to_audio_frames (_video_position - _audio_position)); + emit_silence (_video_position - _audio_position); } } @@ -416,85 +339,103 @@ Player::seek (DCPTime t, bool accurate) } for (list >::iterator i = _pieces.begin(); i != _pieces.end(); ++i) { - shared_ptr vc = dynamic_pointer_cast ((*i)->content); - if (!vc) { - continue; - } - /* s is the offset of t from the start position of this content */ - DCPTime s = t - vc->position (); + DCPTime s = t - (*i)->content->position (); s = max (static_cast (0), s); - s = min (vc->length_after_trim(), s); + s = min ((*i)->content->length_after_trim(), s); - /* Hence set the piece positions to the `global' time */ - (*i)->video_position = (*i)->audio_position = vc->position() + s; + /* Convert this to the content time */ + ContentTime ct = (s * (*i)->frc.speed_up) + (*i)->content->trim_start (); /* And seek the decoder */ - dynamic_pointer_cast((*i)->decoder)->seek (s + vc->trim_start (), accurate); - (*i)->reset_repeat (); + (*i)->decoder->seek (ct, accurate); } _video_position = _audio_position = t; _audio_merger.clear (t); + + if (!accurate) { + /* We just did an inaccurate seek, so it's likely that the next thing seen + out of pass() will be a fair distance from _{video,audio}_position. Setting + this flag stops pass() from trying to fix that: we assume that if it + was an inaccurate seek then the caller does not care too much about + inserting black/silence to keep the time tidy. + */ + _just_did_inaccurate_seek = true; + } } void Player::setup_pieces () { list > old_pieces = _pieces; - _pieces.clear (); ContentList content = _playlist->content (); - sort (content.begin(), content.end(), ContentSorter ()); for (ContentList::iterator i = content.begin(); i != content.end(); ++i) { - shared_ptr piece (new Piece (*i)); - - /* XXX: into content? */ + shared_ptr decoder; + optional frc; shared_ptr fc = dynamic_pointer_cast (*i); if (fc) { - shared_ptr fd (new FFmpegDecoder (_film, fc, _video, _audio)); - - fd->Video.connect (bind (&Player::process_video, this, weak_ptr (piece), _1, _2, _3, _4, 0)); - fd->Audio.connect (bind (&Player::process_audio, this, weak_ptr (piece), _1, _2)); - fd->Subtitle.connect (bind (&Player::process_subtitle, this, weak_ptr (piece), _1, _2, _3, _4)); - - fd->seek (fc->trim_start (), true); - piece->decoder = fd; + decoder.reset (new FFmpegDecoder (_film, fc, _video, _audio)); + frc = FrameRateChange (fc->video_frame_rate(), _film->video_frame_rate()); } shared_ptr ic = dynamic_pointer_cast (*i); if (ic) { - bool reusing = false; - /* See if we can re-use an old ImageDecoder */ for (list >::const_iterator j = old_pieces.begin(); j != old_pieces.end(); ++j) { shared_ptr imd = dynamic_pointer_cast ((*j)->decoder); if (imd && imd->content() == ic) { - piece = *j; - reusing = true; + decoder = imd; } } - if (!reusing) { - shared_ptr id (new ImageDecoder (_film, ic)); - id->Video.connect (bind (&Player::process_video, this, weak_ptr (piece), _1, _2, _3, _4, 0)); - piece->decoder = id; + if (!decoder) { + decoder.reset (new ImageDecoder (_film, ic)); } + + frc = FrameRateChange (ic->video_frame_rate(), _film->video_frame_rate()); } shared_ptr sc = dynamic_pointer_cast (*i); if (sc) { - shared_ptr sd (new SndfileDecoder (_film, sc)); - sd->Audio.connect (bind (&Player::process_audio, this, weak_ptr (piece), _1, _2)); + decoder.reset (new SndfileDecoder (_film, sc)); + + /* Working out the frc for this content is a bit tricky: what if it overlaps + two pieces of video content with different frame rates? For now, use + the one with the best overlap. + */ + + DCPTime best_overlap_t = 0; + shared_ptr best_overlap; + for (ContentList::iterator j = content.begin(); j != content.end(); ++j) { + shared_ptr vc = dynamic_pointer_cast (*j); + if (!vc) { + continue; + } + + DCPTime const overlap = max (vc->position(), sc->position()) - min (vc->end(), sc->end()); + if (overlap > best_overlap_t) { + best_overlap = vc; + best_overlap_t = overlap; + } + } - piece->decoder = sd; + if (best_overlap) { + frc = FrameRateChange (best_overlap->video_frame_rate(), _film->video_frame_rate ()); + } else { + /* No video overlap; e.g. if the DCP is just audio */ + frc = FrameRateChange (_film->video_frame_rate(), _film->video_frame_rate ()); + } } - _pieces.push_back (piece); + decoder->seek ((*i)->trim_start (), true); + + _pieces.push_back (shared_ptr (new Piece (*i, decoder, frc.get ()))); } _have_valid_pieces = true; @@ -596,17 +537,17 @@ Player::emit_black () } void -Player::emit_silence (OutputAudioFrame most) +Player::emit_silence (DCPTime most) { if (most == 0) { return; } - OutputAudioFrame N = min (most, _film->audio_frame_rate() / 2); - shared_ptr silence (new AudioBuffers (_film->audio_channels(), N)); + DCPTime t = min (most, TIME_HZ / 2); + shared_ptr silence (new AudioBuffers (_film->audio_channels(), t * _film->audio_frame_rate() / TIME_HZ)); silence->make_silent (); Audio (silence, _audio_position); - _audio_position += _film->audio_frames_to_time (N); + _audio_position += t; } void @@ -622,18 +563,6 @@ Player::film_changed (Film::Property p) } } -void -Player::process_subtitle (weak_ptr weak_piece, shared_ptr image, dcpomatic::Rect rect, DCPTime from, DCPTime to) -{ - _in_subtitle.piece = weak_piece; - _in_subtitle.image = image; - _in_subtitle.rect = rect; - _in_subtitle.from = from; - _in_subtitle.to = to; - - update_subtitle (); -} - void Player::update_subtitle () { @@ -642,15 +571,15 @@ Player::update_subtitle () return; } - if (!_in_subtitle.image) { - _out_subtitle.image.reset (); + if (!_in_subtitle.subtitle->image) { + _out_subtitle.subtitle->image.reset (); return; } shared_ptr sc = dynamic_pointer_cast (piece->content); assert (sc); - dcpomatic::Rect in_rect = _in_subtitle.rect; + dcpomatic::Rect in_rect = _in_subtitle.subtitle->rect; libdcp::Size scaled_size; in_rect.y += sc->subtitle_offset (); @@ -674,14 +603,15 @@ Player::update_subtitle () _out_subtitle.position.x = rint (_video_container_size.width * (in_rect.x + (in_rect.width * (1 - sc->subtitle_scale ()) / 2))); _out_subtitle.position.y = rint (_video_container_size.height * (in_rect.y + (in_rect.height * (1 - sc->subtitle_scale ()) / 2))); - _out_subtitle.image = _in_subtitle.image->scale ( + _out_subtitle.subtitle->image = _in_subtitle.subtitle->image->scale ( scaled_size, Scaler::from_id ("bicubic"), - _in_subtitle.image->pixel_format (), + _in_subtitle.subtitle->image->pixel_format (), true ); - _out_subtitle.from = _in_subtitle.from + piece->content->position (); - _out_subtitle.to = _in_subtitle.to + piece->content->position (); + + _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time; + _out_subtitle.subtitle->dcp_time = _in_subtitle.subtitle->dcp_time; } /** Re-emit the last frame that was emitted, using current settings for crop, ratio, scaler and subtitles. @@ -690,17 +620,13 @@ Player::update_subtitle () bool Player::repeat_last_video () { - if (!_last_incoming_video.image || !_have_valid_pieces) { + if (!_last_incoming_video.video || !_have_valid_pieces) { return false; } - process_video ( + emit_video ( _last_incoming_video.weak_piece, - _last_incoming_video.image, - _last_incoming_video.eyes, - _last_incoming_video.same, - _last_incoming_video.time, - _last_incoming_video.extra + _last_incoming_video.video ); return true; diff --git a/src/lib/player.h b/src/lib/player.h index 0d6c8d8ba..6e3f8187f 100644 --- a/src/lib/player.h +++ b/src/lib/player.h @@ -29,6 +29,7 @@ #include "rect.h" #include "audio_merger.h" #include "audio_content.h" +#include "decoded.h" class Job; class Film; @@ -38,22 +39,8 @@ class Piece; class Image; class Resampler; -/** @class Player - * @brief A class which can `play' a Playlist; emitting its audio and video. - */ - -struct IncomingVideo -{ -public: - boost::weak_ptr weak_piece; - boost::shared_ptr image; - Eyes eyes; - bool same; - ContentTime time; - ContentTime extra; -}; - -/** A wrapper for an Image which contains some pending operations; these may +/** @class PlayerImage + * @brief A wrapper for an Image which contains some pending operations; these may * not be necessary if the receiver of the PlayerImage throws it away. */ class PlayerImage @@ -75,6 +62,10 @@ private: Position _subtitle_position; }; +/** @class Player + * @brief A class which can `play' a Playlist; emitting its audio and video. + */ + class Player : public boost::enable_shared_from_this, public boost::noncopyable { public: @@ -118,9 +109,6 @@ private: friend class PlayerWrapper; friend class Piece; - void process_video (boost::weak_ptr, boost::shared_ptr, Eyes, bool, ContentTime, ContentTime); - void process_audio (boost::weak_ptr, boost::shared_ptr, ContentTime); - void process_subtitle (boost::weak_ptr, boost::shared_ptr, dcpomatic::Rect, DCPTime, DCPTime); void setup_pieces (); void playlist_changed (); void content_changed (boost::weak_ptr, int, bool); @@ -131,6 +119,8 @@ private: boost::shared_ptr resampler (boost::shared_ptr, bool); void film_changed (Film::Property); void update_subtitle (); + void emit_video (boost::weak_ptr, boost::shared_ptr); + void emit_audio (boost::weak_ptr, boost::shared_ptr); boost::shared_ptr _film; boost::shared_ptr _playlist; @@ -155,17 +145,12 @@ private: struct { boost::weak_ptr piece; - boost::shared_ptr image; - dcpomatic::Rect rect; - DCPTime from; - DCPTime to; + boost::shared_ptr subtitle; } _in_subtitle; struct { - boost::shared_ptr image; Position position; - DCPTime from; - DCPTime to; + boost::shared_ptr subtitle; } _out_subtitle; #ifdef DCPOMATIC_DEBUG @@ -174,7 +159,12 @@ private: bool _last_emit_was_black; - IncomingVideo _last_incoming_video; + struct { + boost::weak_ptr weak_piece; + boost::shared_ptr video; + } _last_incoming_video; + + bool _just_did_inaccurate_seek; boost::signals2::scoped_connection _playlist_changed_connection; boost::signals2::scoped_connection _playlist_content_changed_connection; diff --git a/src/lib/resampler.cc b/src/lib/resampler.cc index d897bf562..00121384d 100644 --- a/src/lib/resampler.cc +++ b/src/lib/resampler.cc @@ -64,11 +64,9 @@ Resampler::~Resampler () swr_free (&_swr_context); } -pair, AudioContent::Frame> -Resampler::run (shared_ptr in, AudioContent::Frame frame) +shared_ptr +Resampler::run (shared_ptr in) { - AudioContent::Frame const resamp_time = swr_next_pts (_swr_context, frame * _out_rate) / _in_rate; - /* Compute the resampled frames count and add 32 for luck */ int const max_resampled_frames = ceil ((double) in->frames() * _out_rate / _in_rate) + 32; shared_ptr resampled (new AudioBuffers (_channels, max_resampled_frames)); @@ -84,7 +82,7 @@ Resampler::run (shared_ptr in, AudioContent::Frame frame) } resampled->set_frames (resampled_frames); - return make_pair (resampled, resamp_time); + return resampled; } shared_ptr diff --git a/src/lib/resampler.h b/src/lib/resampler.h index 69ec83ba9..4ee11a7f0 100644 --- a/src/lib/resampler.h +++ b/src/lib/resampler.h @@ -33,7 +33,7 @@ public: Resampler (int, int, int); ~Resampler (); - std::pair, AudioContent::Frame> run (boost::shared_ptr, AudioContent::Frame); + boost::shared_ptr run (boost::shared_ptr); boost::shared_ptr flush (); private: diff --git a/src/lib/sndfile_decoder.cc b/src/lib/sndfile_decoder.cc index b5a7f139b..3af683c57 100644 --- a/src/lib/sndfile_decoder.cc +++ b/src/lib/sndfile_decoder.cc @@ -55,9 +55,13 @@ SndfileDecoder::~SndfileDecoder () delete[] _deinterleave_buffer; } -void +bool SndfileDecoder::pass () { + if (_remaining == 0) { + return true; + } + /* Do things in half second blocks as I think there may be limits to what FFmpeg (and in particular the resampler) can cope with. */ @@ -93,6 +97,8 @@ SndfileDecoder::pass () audio (data, _done); _done += this_time; _remaining -= this_time; + + return true; } int @@ -113,14 +119,10 @@ SndfileDecoder::audio_frame_rate () const return _info.samplerate; } -bool -SndfileDecoder::done () const -{ - return _audio_position >= _sndfile_content->audio_length (); -} - void -SndfileDecoder::seek (DCPTime t, bool accurate) +SndfileDecoder::seek (ContentTime t, bool accurate) { + Decoder::seek (t, accurate); + /* XXX */ } diff --git a/src/lib/sndfile_decoder.h b/src/lib/sndfile_decoder.h index 470cab956..63f2f7dc4 100644 --- a/src/lib/sndfile_decoder.h +++ b/src/lib/sndfile_decoder.h @@ -29,15 +29,18 @@ public: SndfileDecoder (boost::shared_ptr, boost::shared_ptr); ~SndfileDecoder (); - void pass (); - void seek (DCPTime, bool); - bool done () const; + void seek (ContentTime, bool); int audio_channels () const; AudioContent::Frame audio_length () const; int audio_frame_rate () const; private: + bool has_audio () const { + return true; + } + bool pass (); + boost::shared_ptr _sndfile_content; SNDFILE* _sndfile; SF_INFO _info; diff --git a/src/lib/subtitle_decoder.cc b/src/lib/subtitle_decoder.cc index 8dbc01dc2..389e4b13a 100644 --- a/src/lib/subtitle_decoder.cc +++ b/src/lib/subtitle_decoder.cc @@ -21,6 +21,7 @@ #include "subtitle_decoder.h" using boost::shared_ptr; +using boost::optional; SubtitleDecoder::SubtitleDecoder (shared_ptr f) : Decoder (f) @@ -35,5 +36,5 @@ SubtitleDecoder::SubtitleDecoder (shared_ptr f) void SubtitleDecoder::subtitle (shared_ptr image, dcpomatic::Rect rect, DCPTime from, DCPTime to) { - Subtitle (image, rect, from, to); + _pending.push_back (shared_ptr (new DecodedSubtitle (image, rect, from, to))); } diff --git a/src/lib/subtitle_decoder.h b/src/lib/subtitle_decoder.h index 8a2a0c1b2..4836e31fa 100644 --- a/src/lib/subtitle_decoder.h +++ b/src/lib/subtitle_decoder.h @@ -21,6 +21,7 @@ #include "decoder.h" #include "rect.h" #include "types.h" +#include "decoded.h" class Film; class DCPTimedSubtitle; @@ -31,8 +32,6 @@ class SubtitleDecoder : public virtual Decoder public: SubtitleDecoder (boost::shared_ptr); - boost::signals2::signal, dcpomatic::Rect, DCPTime, DCPTime)> Subtitle; - protected: void subtitle (boost::shared_ptr, dcpomatic::Rect, DCPTime, DCPTime); }; diff --git a/src/lib/video_decoder.cc b/src/lib/video_decoder.cc index 72caf72e9..3a8891111 100644 --- a/src/lib/video_decoder.cc +++ b/src/lib/video_decoder.cc @@ -24,6 +24,7 @@ using std::cout; using boost::shared_ptr; +using boost::optional; VideoDecoder::VideoDecoder (shared_ptr f, shared_ptr c) : Decoder (f) @@ -32,20 +33,20 @@ VideoDecoder::VideoDecoder (shared_ptr f, shared_ptr image, bool same, ContentTime time) { switch (_video_content->video_frame_type ()) { case VIDEO_FRAME_TYPE_2D: - Video (image, EYES_BOTH, same, time); + _pending.push_back (shared_ptr (new DecodedVideo (image, EYES_BOTH, same, time))); break; case VIDEO_FRAME_TYPE_3D_LEFT_RIGHT: { int const half = image->size().width / 2; - Video (image->crop (Crop (0, half, 0, 0), true), EYES_LEFT, same, time); - Video (image->crop (Crop (half, 0, 0, 0), true), EYES_RIGHT, same, time); + _pending.push_back (shared_ptr (new DecodedVideo (image->crop (Crop (0, half, 0, 0), true), EYES_LEFT, same, time))); + _pending.push_back (shared_ptr (new DecodedVideo (image->crop (Crop (half, 0, 0, 0), true), EYES_RIGHT, same, time))); break; } } } - diff --git a/src/lib/video_decoder.h b/src/lib/video_decoder.h index 6e9060dbd..d8c362354 100644 --- a/src/lib/video_decoder.h +++ b/src/lib/video_decoder.h @@ -25,6 +25,7 @@ #include "decoder.h" #include "video_content.h" #include "util.h" +#include "decoded.h" class VideoContent; class Image; @@ -34,14 +35,10 @@ class VideoDecoder : public virtual Decoder public: VideoDecoder (boost::shared_ptr, boost::shared_ptr); - /** Emitted when a video frame is ready. - * First parameter is the video image. - * Second parameter is the eye(s) which should see this image. - * Third parameter is true if the image is the same as the last one that was emitted for this Eyes value. - * Fourth parameter is the time within our source. - */ - boost::signals2::signal, Eyes, bool, ContentTime)> Video; - + boost::shared_ptr video_content () const { + return _video_content; + } + protected: void video (boost::shared_ptr, bool, ContentTime); diff --git a/src/wx/film_editor.cc b/src/wx/film_editor.cc index 14982e0d3..728098b93 100644 --- a/src/wx/film_editor.cc +++ b/src/wx/film_editor.cc @@ -279,7 +279,7 @@ FilmEditor::make_content_panel () b->Add (_content_earlier, 1, wxEXPAND); _content_later = new wxButton (_content_panel, wxID_DOWN); b->Add (_content_later, 1, wxEXPAND); - _content_timeline = new wxButton (_content_panel, wxID_ANY, _("DCPTimeline...")); + _content_timeline = new wxButton (_content_panel, wxID_ANY, _("Timeline...")); b->Add (_content_timeline, 1, wxEXPAND | wxLEFT | wxRIGHT); s->Add (b, 0, wxALL, 4); diff --git a/test/frame_rate_test.cc b/test/frame_rate_test.cc index 7d3904e74..2135b3738 100644 --- a/test/frame_rate_test.cc +++ b/test/frame_rate_test.cc @@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE (best_dcp_frame_rate_test_single) BOOST_CHECK_EQUAL (frc.skip, false); BOOST_CHECK_EQUAL (frc.repeat, 2); BOOST_CHECK_EQUAL (frc.change_speed, true); - BOOST_CHECK_CLOSE (frc.speed_up, 2 * 14.99 / 24, 0.1); + BOOST_CHECK_CLOSE (frc.speed_up, 24 / (2 * 14.99), 0.1); /* Check some conversions with limited DCP targets */ diff --git a/test/play_test.cc b/test/play_test.cc index c9486d161..7beee6f43 100644 --- a/test/play_test.cc +++ b/test/play_test.cc @@ -106,8 +106,6 @@ BOOST_AUTO_TEST_CASE (play_test) shared_ptr player = film->make_player (); PlayerWrapper wrap (player); - /* Seek and audio don't get on at the moment */ - player->disable_audio (); for (int i = 0; i < 32; ++i) { optional