2 Copyright (C) 2013-2021 Carl Hetherington <cth@carlh.net>
4 This file is part of DCP-o-matic.
6 DCP-o-matic is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 DCP-o-matic is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with DCP-o-matic. If not, see <http://www.gnu.org/licenses/>.
22 #include "atmos_decoder.h"
23 #include "audio_buffers.h"
24 #include "audio_content.h"
25 #include "audio_decoder.h"
26 #include "audio_processor.h"
27 #include "compose.hpp"
29 #include "content_audio.h"
30 #include "content_video.h"
31 #include "dcp_content.h"
32 #include "dcp_decoder.h"
33 #include "dcpomatic_log.h"
35 #include "decoder_factory.h"
36 #include "ffmpeg_content.h"
38 #include "frame_rate_change.h"
40 #include "image_decoder.h"
43 #include "maths_util.h"
46 #include "player_video.h"
49 #include "raw_image_proxy.h"
50 #include "render_text.h"
52 #include "text_content.h"
53 #include "text_decoder.h"
55 #include "video_decoder.h"
57 #include <dcp/reel_closed_caption_asset.h>
58 #include <dcp/reel_picture_asset.h>
59 #include <dcp/reel_sound_asset.h>
60 #include <dcp/reel_subtitle_asset.h>
70 using std::dynamic_pointer_cast;
73 using std::make_shared;
74 using std::make_shared;
79 using std::shared_ptr;
82 using boost::optional;
83 using boost::scoped_ptr;
84 #if BOOST_VERSION >= 106100
85 using namespace boost::placeholders;
87 using namespace dcpomatic;
90 int const PlayerProperty::VIDEO_CONTAINER_SIZE = 700;
91 int const PlayerProperty::PLAYLIST = 701;
92 int const PlayerProperty::FILM_CONTAINER = 702;
93 int const PlayerProperty::FILM_VIDEO_FRAME_RATE = 703;
94 int const PlayerProperty::DCP_DECODE_REDUCTION = 704;
95 int const PlayerProperty::PLAYBACK_LENGTH = 705;
96 int const PlayerProperty::IGNORE_VIDEO = 706;
97 int const PlayerProperty::IGNORE_AUDIO = 707;
98 int const PlayerProperty::IGNORE_TEXT = 708;
99 int const PlayerProperty::ALWAYS_BURN_OPEN_SUBTITLES = 709;
100 int const PlayerProperty::PLAY_REFERENCED = 710;
103 Player::Player (shared_ptr<const Film> film, Image::Alignment subtitle_alignment)
106 , _ignore_video(false)
107 , _ignore_audio(false)
108 , _ignore_text(false)
109 , _always_burn_open_subtitles(false)
111 , _tolerant (film->tolerant())
112 , _play_referenced(false)
113 , _audio_merger(film->audio_frame_rate())
114 , _subtitle_alignment (subtitle_alignment)
120 Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist_)
122 , _playlist (playlist_)
124 , _ignore_video(false)
125 , _ignore_audio(false)
126 , _ignore_text(false)
127 , _always_burn_open_subtitles(false)
129 , _tolerant (film->tolerant())
130 , _play_referenced(false)
131 , _audio_merger(film->audio_frame_rate())
140 auto film = _film.lock();
141 DCPOMATIC_ASSERT(film);
144 set_video_container_size(film->frame_size());
146 film_change (ChangeType::DONE, Film::Property::AUDIO_PROCESSOR);
149 seek (DCPTime (), true);
156 auto film = _film.lock();
157 DCPOMATIC_ASSERT(film);
159 _film_changed_connection = film->Change.connect(bind(&Player::film_change, this, _1, _2));
160 /* The butler must hear about this first, so since we are proxying this through to the butler we must
163 _playlist_change_connection = playlist()->Change.connect (bind (&Player::playlist_change, this, _1), boost::signals2::at_front);
164 _playlist_content_change_connection = playlist()->ContentChange.connect (bind(&Player::playlist_content_change, this, _1, _3, _4));
168 Player::Player(Player&& other)
170 , _playlist(std::move(other._playlist))
171 , _suspended(other._suspended.load())
172 , _pieces(std::move(other._pieces))
173 , _video_container_size(other._video_container_size.load())
174 , _black_image(std::move(other._black_image))
175 , _ignore_video(other._ignore_video.load())
176 , _ignore_audio(other._ignore_audio.load())
177 , _ignore_text(other._ignore_text.load())
178 , _always_burn_open_subtitles(other._always_burn_open_subtitles.load())
179 , _fast(other._fast.load())
180 , _tolerant(other._tolerant)
181 , _play_referenced(other._play_referenced.load())
182 , _next_video_time(other._next_video_time)
183 , _next_audio_time(other._next_audio_time)
184 , _dcp_decode_reduction(other._dcp_decode_reduction.load())
185 , _last_video(std::move(other._last_video))
186 , _audio_merger(std::move(other._audio_merger))
187 , _shuffler(std::move(other._shuffler))
188 , _delay(std::move(other._delay))
189 , _stream_states(std::move(other._stream_states))
190 , _black(std::move(other._black))
191 , _silent(std::move(other._silent))
192 , _active_texts(std::move(other._active_texts))
193 , _audio_processor(std::move(other._audio_processor))
194 , _playback_length(other._playback_length.load())
195 , _subtitle_alignment(other._subtitle_alignment)
202 Player::operator=(Player&& other)
204 if (this == &other) {
208 _film = std::move(other._film);
209 _playlist = std::move(other._playlist);
210 _suspended = other._suspended.load();
211 _pieces = std::move(other._pieces);
212 _video_container_size = other._video_container_size.load();
213 _black_image = std::move(other._black_image);
214 _ignore_video = other._ignore_video.load();
215 _ignore_audio = other._ignore_audio.load();
216 _ignore_text = other._ignore_text.load();
217 _always_burn_open_subtitles = other._always_burn_open_subtitles.load();
218 _fast = other._fast.load();
219 _tolerant = other._tolerant;
220 _play_referenced = other._play_referenced.load();
221 _next_video_time = other._next_video_time;
222 _next_audio_time = other._next_audio_time;
223 _dcp_decode_reduction = other._dcp_decode_reduction.load();
224 _last_video = std::move(other._last_video);
225 _audio_merger = std::move(other._audio_merger);
226 _shuffler = std::move(other._shuffler);
227 _delay = std::move(other._delay);
228 _stream_states = std::move(other._stream_states);
229 _black = std::move(other._black);
230 _silent = std::move(other._silent);
231 _active_texts = std::move(other._active_texts);
232 _audio_processor = std::move(other._audio_processor);
233 _playback_length = other._playback_length.load();
234 _subtitle_alignment = other._subtitle_alignment;
243 have_video (shared_ptr<const Content> content)
245 return static_cast<bool>(content->video) && content->video->use() && content->can_be_played();
250 have_audio (shared_ptr<const Content> content)
252 return static_cast<bool>(content->audio) && content->can_be_played();
257 Player::setup_pieces ()
259 boost::mutex::scoped_lock lm (_mutex);
261 auto old_pieces = _pieces;
264 auto film = _film.lock();
269 _playback_length = _playlist ? _playlist->length(film) : film->length();
271 auto playlist_content = playlist()->content();
272 bool const have_threed = std::any_of(
273 playlist_content.begin(),
274 playlist_content.end(),
275 [](shared_ptr<const Content> c) {
276 return c->video && (c->video->frame_type() == VideoFrameType::THREE_D_LEFT || c->video->frame_type() == VideoFrameType::THREE_D_RIGHT);
281 _shuffler.reset(new Shuffler());
282 _shuffler->Video.connect(bind(&Player::video, this, _1, _2));
285 for (auto content: playlist()->content()) {
287 if (!content->paths_valid()) {
291 if (_ignore_video && _ignore_audio && content->text.empty()) {
292 /* We're only interested in text and this content has none */
296 shared_ptr<Decoder> old_decoder;
297 for (auto j: old_pieces) {
298 if (j->content == content) {
299 old_decoder = j->decoder;
304 auto decoder = decoder_factory(film, content, _fast, _tolerant, old_decoder);
305 DCPOMATIC_ASSERT (decoder);
307 FrameRateChange frc(film, content);
309 if (decoder->video && _ignore_video) {
310 decoder->video->set_ignore (true);
313 if (decoder->audio && _ignore_audio) {
314 decoder->audio->set_ignore (true);
318 for (auto i: decoder->text) {
319 i->set_ignore (true);
323 auto dcp = dynamic_pointer_cast<DCPDecoder> (decoder);
325 dcp->set_decode_referenced (_play_referenced);
326 if (_play_referenced) {
327 dcp->set_forced_reduction (_dcp_decode_reduction);
331 auto piece = make_shared<Piece>(content, decoder, frc);
332 _pieces.push_back (piece);
334 if (decoder->video) {
336 /* We need a Shuffler to cope with 3D L/R video data arriving out of sequence */
337 decoder->video->Data.connect (bind(&Shuffler::video, _shuffler.get(), weak_ptr<Piece>(piece), _1));
339 decoder->video->Data.connect (bind(&Player::video, this, weak_ptr<Piece>(piece), _1));
343 if (decoder->audio) {
344 decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
347 auto j = decoder->text.begin();
349 while (j != decoder->text.end()) {
350 (*j)->BitmapStart.connect (
351 bind(&Player::bitmap_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
353 (*j)->PlainStart.connect (
354 bind(&Player::plain_text_start, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
357 bind(&Player::subtitle_stop, this, weak_ptr<Piece>(piece), weak_ptr<const TextContent>((*j)->content()), _1)
363 if (decoder->atmos) {
364 decoder->atmos->Data.connect (bind(&Player::atmos, this, weak_ptr<Piece>(piece), _1));
368 _stream_states.clear ();
369 for (auto i: _pieces) {
370 if (i->content->audio) {
371 for (auto j: i->content->audio->streams()) {
372 _stream_states[j] = StreamState(i);
377 auto ignore_overlap = [](shared_ptr<VideoContent> v) {
378 return v && v->use() && v->frame_type() != VideoFrameType::THREE_D_LEFT && v->frame_type() != VideoFrameType::THREE_D_RIGHT;
381 for (auto piece = _pieces.begin(); piece != _pieces.end(); ++piece) {
382 if (ignore_overlap((*piece)->content->video)) {
383 /* Look for content later in the content list with in-use video that overlaps this */
384 auto const period = (*piece)->content->period(film);
385 for (auto later_piece = std::next(piece); later_piece != _pieces.end(); ++later_piece) {
386 if (ignore_overlap((*later_piece)->content->video)) {
387 if (auto overlap = (*later_piece)->content->period(film).overlap(period)) {
388 (*piece)->ignore_video.push_back(*overlap);
395 for (auto piece = _pieces.begin(); piece != _pieces.end(); ++piece) {
396 if ((*piece)->content->atmos) {
397 /* Look for content later in the content list with ATMOS that overlaps this */
398 auto const period = (*piece)->content->period(film);
399 for (auto later_piece = std::next(piece); later_piece != _pieces.end(); ++later_piece) {
400 if ((*later_piece)->content->atmos) {
401 if (auto overlap = (*later_piece)->content->period(film).overlap(period)) {
402 (*piece)->ignore_atmos.push_back(*overlap);
409 _black = Empty(film, playlist(), bind(&have_video, _1), _playback_length);
410 _silent = Empty(film, playlist(), bind(&have_audio, _1), _playback_length);
412 _next_video_time = boost::none;
413 _next_video_eyes = Eyes::BOTH;
414 _next_audio_time = boost::none;
419 Player::playlist_content_change (ChangeType type, int property, bool frequent)
421 auto film = _film.lock();
426 if (property == VideoContentProperty::CROP) {
427 if (type == ChangeType::DONE) {
428 boost::mutex::scoped_lock lm (_mutex);
429 for (auto const& i: _delay) {
430 i.first->reset_metadata(film, _video_container_size);
434 if (type == ChangeType::PENDING) {
435 /* The player content is probably about to change, so we can't carry on
436 until that has happened and we've rebuilt our pieces. Stop pass()
437 and seek() from working until then.
440 } else if (type == ChangeType::DONE) {
441 /* A change in our content has gone through. Re-build our pieces. */
444 } else if (type == ChangeType::CANCELLED) {
449 Change (type, property, frequent);
454 Player::set_video_container_size (dcp::Size s)
456 ChangeSignaller<Player, int> cc(this, PlayerProperty::VIDEO_CONTAINER_SIZE);
458 if (s == _video_container_size) {
463 _video_container_size = s;
466 boost::mutex::scoped_lock lm(_black_image_mutex);
467 _black_image = make_shared<Image>(AV_PIX_FMT_RGB24, _video_container_size, Image::Alignment::PADDED);
468 _black_image->make_black ();
474 Player::playlist_change (ChangeType type)
476 if (type == ChangeType::DONE) {
479 Change (type, PlayerProperty::PLAYLIST, false);
484 Player::film_change (ChangeType type, Film::Property p)
486 /* Here we should notice Film properties that affect our output, and
487 alert listeners that our output now would be different to how it was
488 last time we were run.
491 auto film = _film.lock();
496 if (p == Film::Property::CONTAINER) {
497 Change (type, PlayerProperty::FILM_CONTAINER, false);
498 } else if (p == Film::Property::VIDEO_FRAME_RATE) {
499 /* Pieces contain a FrameRateChange which contains the DCP frame rate,
500 so we need new pieces here.
502 if (type == ChangeType::DONE) {
505 Change (type, PlayerProperty::FILM_VIDEO_FRAME_RATE, false);
506 } else if (p == Film::Property::AUDIO_PROCESSOR) {
507 if (type == ChangeType::DONE && film->audio_processor ()) {
508 boost::mutex::scoped_lock lm (_mutex);
509 _audio_processor = film->audio_processor()->clone(film->audio_frame_rate());
511 } else if (p == Film::Property::AUDIO_CHANNELS) {
512 if (type == ChangeType::DONE) {
513 boost::mutex::scoped_lock lm (_mutex);
514 _audio_merger.clear ();
520 shared_ptr<PlayerVideo>
521 Player::black_player_video_frame (Eyes eyes) const
523 boost::mutex::scoped_lock lm(_black_image_mutex);
525 return std::make_shared<PlayerVideo> (
526 std::make_shared<const RawImageProxy>(_black_image),
529 _video_container_size,
530 _video_container_size,
533 PresetColourConversion::all().front().conversion,
535 std::weak_ptr<Content>(),
536 boost::optional<Frame>(),
543 Player::dcp_to_content_video (shared_ptr<const Piece> piece, DCPTime t) const
545 auto film = _film.lock();
546 DCPOMATIC_ASSERT(film);
548 auto s = t - piece->content->position ();
549 s = min (piece->content->length_after_trim(film), s);
550 s = max (DCPTime(), s + DCPTime (piece->content->trim_start(), piece->frc));
552 /* It might seem more logical here to convert s to a ContentTime (using the FrameRateChange)
553 then convert that ContentTime to frames at the content's rate. However this fails for
554 situations like content at 29.9978733fps, DCP at 30fps. The accuracy of the Time type is not
555 enough to distinguish between the two with low values of time (e.g. 3200 in Time units).
557 Instead we convert the DCPTime using the DCP video rate then account for any skip/repeat.
559 return s.frames_floor (piece->frc.dcp) / piece->frc.factor ();
564 Player::content_video_to_dcp (shared_ptr<const Piece> piece, Frame f) const
566 /* See comment in dcp_to_content_video */
567 auto const d = DCPTime::from_frames (f * piece->frc.factor(), piece->frc.dcp) - DCPTime(piece->content->trim_start(), piece->frc);
568 return d + piece->content->position();
573 Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
575 auto film = _film.lock();
576 DCPOMATIC_ASSERT(film);
578 auto s = t - piece->content->position ();
579 s = min (piece->content->length_after_trim(film), s);
580 /* See notes in dcp_to_content_video */
581 return max (DCPTime(), DCPTime(piece->content->trim_start(), piece->frc) + s).frames_floor(film->audio_frame_rate());
586 Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
588 auto film = _film.lock();
589 DCPOMATIC_ASSERT(film);
591 /* See comment in dcp_to_content_video */
592 return DCPTime::from_frames(f, film->audio_frame_rate())
593 - DCPTime (piece->content->trim_start(), piece->frc)
594 + piece->content->position();
599 Player::dcp_to_content_time (shared_ptr<const Piece> piece, DCPTime t) const
601 auto film = _film.lock();
602 DCPOMATIC_ASSERT(film);
604 auto s = t - piece->content->position ();
605 s = min (piece->content->length_after_trim(film), s);
606 return max (ContentTime (), ContentTime (s, piece->frc) + piece->content->trim_start());
611 Player::content_time_to_dcp (shared_ptr<const Piece> piece, ContentTime t) const
613 return max (DCPTime(), DCPTime(t - piece->content->trim_start(), piece->frc) + piece->content->position());
617 vector<shared_ptr<Font>>
618 Player::get_subtitle_fonts ()
620 boost::mutex::scoped_lock lm (_mutex);
622 vector<shared_ptr<Font>> fonts;
623 for (auto piece: _pieces) {
624 for (auto text: piece->content->text) {
625 auto text_fonts = text->fonts();
626 copy (text_fonts.begin(), text_fonts.end(), back_inserter(fonts));
634 /** Set this player never to produce any video data */
636 Player::set_ignore_video ()
638 ChangeSignaller<Player, int> cc(this, PlayerProperty::IGNORE_VIDEO);
639 _ignore_video = true;
645 Player::set_ignore_audio ()
647 ChangeSignaller<Player, int> cc(this, PlayerProperty::IGNORE_AUDIO);
648 _ignore_audio = true;
654 Player::set_ignore_text ()
656 ChangeSignaller<Player, int> cc(this, PlayerProperty::IGNORE_TEXT);
662 /** Set the player to always burn open texts into the image regardless of the content settings */
664 Player::set_always_burn_open_subtitles ()
666 ChangeSignaller<Player, int> cc(this, PlayerProperty::ALWAYS_BURN_OPEN_SUBTITLES);
667 _always_burn_open_subtitles = true;
671 /** Sets up the player to be faster, possibly at the expense of quality */
681 Player::set_play_referenced ()
683 ChangeSignaller<Player, int> cc(this, PlayerProperty::PLAY_REFERENCED);
684 _play_referenced = true;
692 boost::mutex::scoped_lock lm (_mutex);
695 /* We can't pass in this state */
696 LOG_DEBUG_PLAYER_NC ("Player is suspended");
700 auto film = _film.lock();
702 if (_playback_length.load() == DCPTime() || !film) {
703 /* Special; just give one black frame */
704 emit_video (black_player_video_frame(Eyes::BOTH), DCPTime());
708 /* Find the decoder or empty which is farthest behind where we are and make it emit some data */
710 shared_ptr<Piece> earliest_content;
711 optional<DCPTime> earliest_time;
713 for (auto i: _pieces) {
718 auto const t = content_time_to_dcp (i, max(i->decoder->position(), i->content->trim_start()));
719 if (t > i->content->end(film)) {
723 /* Given two choices at the same time, pick the one with texts so we see it before
726 if (!earliest_time || t < *earliest_time || (t == *earliest_time && !i->decoder->text.empty())) {
728 earliest_content = i;
742 if (earliest_content) {
746 if (!_black.done() && !_ignore_video && (!earliest_time || _black.position() < *earliest_time)) {
747 earliest_time = _black.position ();
751 if (!_silent.done() && !_ignore_audio && (!earliest_time || _silent.position() < *earliest_time)) {
752 earliest_time = _silent.position ();
759 LOG_DEBUG_PLAYER ("Calling pass() on %1", earliest_content->content->path(0));
760 earliest_content->done = earliest_content->decoder->pass ();
761 auto dcp = dynamic_pointer_cast<DCPContent>(earliest_content->content);
762 if (dcp && !_play_referenced && dcp->reference_audio()) {
763 /* We are skipping some referenced DCP audio content, so we need to update _next_audio_time
764 to `hide' the fact that no audio was emitted during the referenced DCP (though
765 we need to behave as though it was).
767 _next_audio_time = dcp->end(film);
772 LOG_DEBUG_PLAYER ("Emit black for gap at %1", to_string(_black.position()));
773 emit_video (black_player_video_frame(Eyes::BOTH), _black.position());
774 _black.set_position (_black.position() + one_video_frame());
778 LOG_DEBUG_PLAYER ("Emit silence for gap at %1", to_string(_silent.position()));
779 DCPTimePeriod period (_silent.period_at_position());
780 if (_next_audio_time) {
781 /* Sometimes the thing that happened last finishes fractionally before
782 or after this silence. Bodge the start time of the silence to fix it.
783 I think this is nothing to worry about since we will just add or
784 remove a little silence at the end of some content.
786 int64_t const error = labs(period.from.get() - _next_audio_time->get());
787 /* Let's not worry about less than a frame at 24fps */
788 int64_t const too_much_error = DCPTime::from_frames(1, 24).get();
789 if (error >= too_much_error) {
790 film->log()->log(String::compose("Silence starting before or after last audio by %1", error), LogEntry::TYPE_ERROR);
792 DCPOMATIC_ASSERT (error < too_much_error);
793 period.from = *_next_audio_time;
795 if (period.duration() > one_video_frame()) {
796 period.to = period.from + one_video_frame();
799 _silent.set_position (period.to);
807 /* Emit any audio that is ready */
809 /* Work out the time before which the audio is definitely all here. This is the earliest last_push_end of one
810 of our streams, or the position of the _silent. First, though we choose only streams that are less than
811 ignore_streams_behind seconds behind the furthest ahead (we assume that if a stream has fallen that far
812 behind it has finished). This is so that we don't withhold audio indefinitely awaiting data from a stream
813 that will never come, causing bugs like #2101.
815 constexpr int ignore_streams_behind = 5;
817 using state_pair = std::pair<AudioStreamPtr, StreamState>;
819 /* Find streams that have pushed */
820 std::vector<state_pair> have_pushed;
821 std::copy_if(_stream_states.begin(), _stream_states.end(), std::back_inserter(have_pushed), [](state_pair const& a) { return static_cast<bool>(a.second.last_push_end); });
823 /* Find the 'leading' stream (i.e. the one that pushed data most recently) */
824 auto latest_last_push_end = std::max_element(
827 [](state_pair const& a, state_pair const& b) { return a.second.last_push_end.get() < b.second.last_push_end.get(); }
830 if (latest_last_push_end != have_pushed.end()) {
831 LOG_DEBUG_PLAYER("Leading audio stream is in %1 at %2", latest_last_push_end->second.piece->content->path(0), to_string(latest_last_push_end->second.last_push_end.get()));
834 /* Now make a list of those streams that are less than ignore_streams_behind behind the leader */
835 std::map<AudioStreamPtr, StreamState> alive_stream_states;
836 for (auto const& i: _stream_states) {
837 if (!i.second.last_push_end || (latest_last_push_end->second.last_push_end.get() - i.second.last_push_end.get()) < dcpomatic::DCPTime::from_seconds(ignore_streams_behind)) {
838 alive_stream_states.insert(i);
840 LOG_DEBUG_PLAYER("Ignoring stream %1 because it is too far behind", i.second.piece->content->path(0));
844 auto pull_to = _playback_length.load();
845 for (auto const& i: alive_stream_states) {
846 auto position = i.second.last_push_end.get_value_or(i.second.piece->content->position());
847 if (!i.second.piece->done && position < pull_to) {
851 if (!_silent.done() && _silent.position() < pull_to) {
852 pull_to = _silent.position();
855 LOG_DEBUG_PLAYER("Emitting audio up to %1", to_string(pull_to));
856 auto audio = _audio_merger.pull (pull_to);
857 for (auto i = audio.begin(); i != audio.end(); ++i) {
858 if (_next_audio_time && i->second < *_next_audio_time) {
859 /* This new data comes before the last we emitted (or the last seek); discard it */
860 auto cut = discard_audio (i->first, i->second, *_next_audio_time);
865 } else if (_next_audio_time && i->second > *_next_audio_time) {
866 /* There's a gap between this data and the last we emitted; fill with silence */
867 fill_audio (DCPTimePeriod (*_next_audio_time, i->second));
870 emit_audio (i->first, i->second);
877 for (auto const& i: _delay) {
878 do_emit_video(i.first, i.second);
881 /* Perhaps we should have Empty entries for both eyes in the 3D case (somehow).
882 * However, if we have L and R video files, and one is shorter than the other,
883 * the fill code in ::video mostly takes care of filling in the gaps.
884 * However, since it fills at the point when it knows there is more video coming
885 * at time t (so it should fill any gap up to t) it can't do anything right at the
886 * end. This is particularly bad news if the last frame emitted is a LEFT
887 * eye, as the MXF writer will complain about the 3D sequence being wrong.
888 * Here's a hack to workaround that particular case.
890 if (_next_video_eyes && _next_video_time && *_next_video_eyes == Eyes::RIGHT) {
891 do_emit_video (black_player_video_frame(Eyes::RIGHT), *_next_video_time);
899 /** @return Open subtitles for the frame at the given time, converted to images */
900 optional<PositionImage>
901 Player::open_subtitles_for_frame (DCPTime time) const
903 auto film = _film.lock();
908 list<PositionImage> captions;
909 int const vfr = film->video_frame_rate();
913 _active_texts[TextType::OPEN_SUBTITLE].get_burnt(DCPTimePeriod(time, time + DCPTime::from_frames(1, vfr)), _always_burn_open_subtitles)
916 /* Bitmap subtitles */
917 for (auto i: j.bitmap) {
922 /* i.image will already have been scaled to fit _video_container_size */
923 dcp::Size scaled_size (i.rectangle.width * _video_container_size.load().width, i.rectangle.height * _video_container_size.load().height);
929 lrint(_video_container_size.load().width * i.rectangle.x),
930 lrint(_video_container_size.load().height * i.rectangle.y)
936 /* String subtitles (rendered to an image) */
937 if (!j.string.empty()) {
938 auto s = render_text(j.string, _video_container_size, time, vfr);
939 copy (s.begin(), s.end(), back_inserter (captions));
943 if (captions.empty()) {
947 return merge (captions, _subtitle_alignment);
953 increment_eyes (Eyes e)
955 if (e == Eyes::LEFT) {
964 Player::video (weak_ptr<Piece> weak_piece, ContentVideo video)
970 auto piece = weak_piece.lock ();
975 if (!piece->content->video->use()) {
979 auto film = _film.lock();
984 FrameRateChange frc(film, piece->content);
985 if (frc.skip && (video.frame % 2) == 1) {
989 vector<Eyes> eyes_to_emit;
991 if (!film->three_d()) {
992 if (video.eyes == Eyes::RIGHT) {
993 /* 2D film, 3D content: discard right */
995 } else if (video.eyes == Eyes::LEFT) {
996 /* 2D film, 3D content: emit left as "both" */
997 video.eyes = Eyes::BOTH;
998 eyes_to_emit = { Eyes::BOTH };
1001 if (video.eyes == Eyes::BOTH) {
1002 /* 3D film, 2D content; emit "both" for left and right */
1003 eyes_to_emit = { Eyes::LEFT, Eyes::RIGHT };
1007 if (eyes_to_emit.empty()) {
1008 eyes_to_emit = { video.eyes };
1011 /* Time of the first frame we will emit */
1012 DCPTime const time = content_video_to_dcp (piece, video.frame);
1013 LOG_DEBUG_PLAYER("Received video frame %1 at %2", video.frame, to_string(time));
1015 /* Discard if it's before the content's period or the last accurate seek. We can't discard
1016 if it's after the content's period here as in that case we still need to fill any gap between
1017 `now' and the end of the content's period.
1019 if (time < piece->content->position() || (_next_video_time && time < *_next_video_time)) {
1023 auto ignore_video = std::find_if(
1024 piece->ignore_video.begin(),
1025 piece->ignore_video.end(),
1026 [time](DCPTimePeriod period) { return period.contains(time); }
1028 if (ignore_video != piece->ignore_video.end()) {
1032 /* Fill gaps that we discover now that we have some video which needs to be emitted.
1033 This is where we need to fill to.
1035 DCPTime fill_to = min(time, piece->content->end(film));
1037 if (_next_video_time) {
1038 DCPTime fill_from = max (*_next_video_time, piece->content->position());
1040 /* Fill if we have more than half a frame to do */
1041 if ((fill_to - fill_from) > one_video_frame() / 2) {
1042 auto last = _last_video.find (weak_piece);
1043 if (film->three_d()) {
1044 auto fill_to_eyes = eyes_to_emit[0];
1045 if (fill_to_eyes == Eyes::BOTH) {
1046 fill_to_eyes = Eyes::LEFT;
1048 if (fill_to == piece->content->end(film)) {
1049 /* Don't fill after the end of the content */
1050 fill_to_eyes = Eyes::LEFT;
1053 auto eyes = _next_video_eyes.get_value_or(Eyes::LEFT);
1054 if (eyes == Eyes::BOTH) {
1057 while (j < fill_to || eyes != fill_to_eyes) {
1058 if (last != _last_video.end()) {
1059 LOG_DEBUG_PLAYER("Fill using last video at %1 in 3D mode", to_string(j));
1060 auto copy = last->second->shallow_copy();
1061 copy->set_eyes (eyes);
1062 emit_video (copy, j);
1064 LOG_DEBUG_PLAYER("Fill using black at %1 in 3D mode", to_string(j));
1065 emit_video (black_player_video_frame(eyes), j);
1067 if (eyes == Eyes::RIGHT) {
1068 j += one_video_frame();
1070 eyes = increment_eyes (eyes);
1073 for (DCPTime j = fill_from; j < fill_to; j += one_video_frame()) {
1074 if (last != _last_video.end()) {
1075 emit_video (last->second, j);
1077 emit_video (black_player_video_frame(Eyes::BOTH), j);
1084 auto const content_video = piece->content->video;
1086 for (auto eyes: eyes_to_emit) {
1087 _last_video[weak_piece] = std::make_shared<PlayerVideo>(
1089 content_video->actual_crop(),
1090 content_video->fade(film, video.frame),
1092 content_video->scaled_size(film->frame_size()),
1093 _video_container_size,
1095 content_video->pixel_quanta()
1097 _video_container_size,
1100 content_video->colour_conversion(),
1101 content_video->range(),
1108 for (int i = 0; i < frc.repeat; ++i) {
1109 if (t < piece->content->end(film)) {
1110 emit_video (_last_video[weak_piece], t);
1112 t += one_video_frame ();
1119 Player::audio (weak_ptr<Piece> weak_piece, AudioStreamPtr stream, ContentAudio content_audio)
1125 DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
1127 auto piece = weak_piece.lock ();
1132 auto film = _film.lock();
1137 auto content = piece->content->audio;
1138 DCPOMATIC_ASSERT (content);
1140 int const rfr = content->resampled_frame_rate(film);
1142 /* Compute time in the DCP */
1143 auto time = resampled_audio_to_dcp (piece, content_audio.frame);
1145 /* And the end of this block in the DCP */
1146 auto end = time + DCPTime::from_frames(content_audio.audio->frames(), rfr);
1147 LOG_DEBUG_PLAYER("Received audio frame %1 covering %2 to %3 (%4)", content_audio.frame, to_string(time), to_string(end), piece->content->path(0).filename());
1149 /* Remove anything that comes before the start or after the end of the content */
1150 if (time < piece->content->position()) {
1151 auto cut = discard_audio (content_audio.audio, time, piece->content->position());
1153 /* This audio is entirely discarded */
1156 content_audio.audio = cut.first;
1158 } else if (time > piece->content->end(film)) {
1159 /* Discard it all */
1161 } else if (end > piece->content->end(film)) {
1162 Frame const remaining_frames = DCPTime(piece->content->end(film) - time).frames_round(rfr);
1163 if (remaining_frames == 0) {
1166 content_audio.audio = make_shared<AudioBuffers>(content_audio.audio, remaining_frames, 0);
1169 DCPOMATIC_ASSERT (content_audio.audio->frames() > 0);
1173 auto const fade_coeffs = content->fade (stream, content_audio.frame, content_audio.audio->frames(), rfr);
1174 if (content->gain() != 0 || !fade_coeffs.empty()) {
1175 auto gain_buffers = make_shared<AudioBuffers>(content_audio.audio);
1176 if (!fade_coeffs.empty()) {
1177 /* Apply both fade and gain */
1178 DCPOMATIC_ASSERT (fade_coeffs.size() == static_cast<size_t>(gain_buffers->frames()));
1179 auto const channels = gain_buffers->channels();
1180 auto const frames = fade_coeffs.size();
1181 auto data = gain_buffers->data();
1182 auto const gain = db_to_linear (content->gain());
1183 for (auto channel = 0; channel < channels; ++channel) {
1184 for (auto frame = 0U; frame < frames; ++frame) {
1185 data[channel][frame] *= gain * fade_coeffs[frame];
1189 /* Just apply gain */
1190 gain_buffers->apply_gain (content->gain());
1192 content_audio.audio = gain_buffers;
1197 content_audio.audio = remap(content_audio.audio, film->audio_channels(), stream->mapping());
1201 if (_audio_processor) {
1202 content_audio.audio = _audio_processor->run(content_audio.audio, film->audio_channels());
1207 _audio_merger.push (content_audio.audio, time);
1208 DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
1209 _stream_states[stream].last_push_end = time + DCPTime::from_frames(content_audio.audio->frames(), film->audio_frame_rate());
1214 Player::bitmap_text_start (weak_ptr<Piece> weak_piece, weak_ptr<const TextContent> weak_content, ContentBitmapText subtitle)
1220 auto piece = weak_piece.lock ();
1221 auto content = weak_content.lock ();
1222 if (!piece || !content) {
1227 for (auto& sub: subtitle.subs)
1229 /* Apply content's subtitle offsets */
1230 sub.rectangle.x += content->x_offset ();
1231 sub.rectangle.y += content->y_offset ();
1233 /* Apply a corrective translation to keep the subtitle centred after the scale that is coming up */
1234 sub.rectangle.x -= sub.rectangle.width * ((content->x_scale() - 1) / 2);
1235 sub.rectangle.y -= sub.rectangle.height * ((content->y_scale() - 1) / 2);
1237 /* Apply content's subtitle scale */
1238 sub.rectangle.width *= content->x_scale ();
1239 sub.rectangle.height *= content->y_scale ();
1241 auto image = sub.image;
1243 /* We will scale the subtitle up to fit _video_container_size */
1244 int const width = sub.rectangle.width * _video_container_size.load().width;
1245 int const height = sub.rectangle.height * _video_container_size.load().height;
1246 if (width == 0 || height == 0) {
1250 dcp::Size scaled_size (width, height);
1251 ps.bitmap.push_back (BitmapText(image->scale(scaled_size, dcp::YUVToRGB::REC601, image->pixel_format(), Image::Alignment::PADDED, _fast), sub.rectangle));
1254 DCPTime from(content_time_to_dcp(piece, subtitle.from()));
1255 _active_texts[content->type()].add_from(weak_content, ps, from);
1260 Player::plain_text_start (weak_ptr<Piece> weak_piece, weak_ptr<const TextContent> weak_content, ContentStringText subtitle)
1266 auto piece = weak_piece.lock ();
1267 auto content = weak_content.lock ();
1268 auto film = _film.lock();
1269 if (!piece || !content || !film) {
1274 DCPTime const from (content_time_to_dcp (piece, subtitle.from()));
1276 if (from > piece->content->end(film)) {
1280 for (auto s: subtitle.subs) {
1281 s.set_h_position (s.h_position() + content->x_offset());
1282 s.set_v_position (s.v_position() + content->y_offset());
1283 float const xs = content->x_scale();
1284 float const ys = content->y_scale();
1285 float size = s.size();
1287 /* Adjust size to express the common part of the scaling;
1288 e.g. if xs = ys = 0.5 we scale size by 2.
1290 if (xs > 1e-5 && ys > 1e-5) {
1291 size *= 1 / min (1 / xs, 1 / ys);
1295 /* Then express aspect ratio changes */
1296 if (fabs (1.0 - xs / ys) > dcp::ASPECT_ADJUST_EPSILON) {
1297 s.set_aspect_adjust (xs / ys);
1300 s.set_in (dcp::Time(from.seconds(), 1000));
1301 ps.string.push_back (s);
1304 _active_texts[content->type()].add_from(weak_content, ps, from);
1309 Player::subtitle_stop (weak_ptr<Piece> weak_piece, weak_ptr<const TextContent> weak_content, ContentTime to)
1315 auto content = weak_content.lock ();
1320 if (!_active_texts[content->type()].have(weak_content)) {
1324 auto piece = weak_piece.lock ();
1325 auto film = _film.lock();
1326 if (!piece || !film) {
1330 DCPTime const dcp_to = content_time_to_dcp (piece, to);
1332 if (dcp_to > piece->content->end(film)) {
1336 auto from = _active_texts[content->type()].add_to(weak_content, dcp_to);
1338 bool const always = (content->type() == TextType::OPEN_SUBTITLE && _always_burn_open_subtitles);
1339 if (content->use() && !always && !content->burn()) {
1340 Text (from.first, content->type(), content->dcp_track().get_value_or(DCPTextTrack()), DCPTimePeriod(from.second, dcp_to));
1346 Player::seek (DCPTime time, bool accurate)
1348 boost::mutex::scoped_lock lm (_mutex);
1349 LOG_DEBUG_PLAYER("Seek to %1 (%2accurate)", to_string(time), accurate ? "" : "in");
1352 /* We can't seek in this state */
1356 auto film = _film.lock();
1362 _shuffler->clear ();
1367 if (_audio_processor) {
1368 _audio_processor->flush ();
1371 _audio_merger.clear ();
1372 std::for_each(_active_texts.begin(), _active_texts.end(), [](ActiveText& a) { a.clear(); });
1374 for (auto i: _pieces) {
1375 if (time < i->content->position()) {
1376 /* Before; seek to the start of the content. Even if this request is for an inaccurate seek
1377 we must seek this (following) content accurately, otherwise when we come to the end of the current
1378 content we may not start right at the beginning of the next, causing a gap (if the next content has
1379 been trimmed to a point between keyframes, or something).
1381 i->decoder->seek (dcp_to_content_time (i, i->content->position()), true);
1383 } else if (i->content->position() <= time && time < i->content->end(film)) {
1384 /* During; seek to position */
1385 i->decoder->seek (dcp_to_content_time (i, time), accurate);
1388 /* After; this piece is done */
1394 _next_video_time = time;
1395 _next_video_eyes = Eyes::LEFT;
1396 _next_audio_time = time;
1398 _next_video_time = boost::none;
1399 _next_video_eyes = boost::none;
1400 _next_audio_time = boost::none;
1403 _black.set_position (time);
1404 _silent.set_position (time);
1406 _last_video.clear ();
1408 for (auto& state: _stream_states) {
1409 state.second.last_push_end = boost::none;
1415 Player::emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1417 auto film = _film.lock();
1418 DCPOMATIC_ASSERT(film);
1420 if (!film->three_d()) {
1421 if (pv->eyes() == Eyes::LEFT) {
1422 /* Use left-eye images for both eyes... */
1423 pv->set_eyes (Eyes::BOTH);
1424 } else if (pv->eyes() == Eyes::RIGHT) {
1425 /* ...and discard the right */
1430 /* We need a delay to give a little wiggle room to ensure that relevant subtitles arrive at the
1431 player before the video that requires them.
1433 _delay.push_back (make_pair (pv, time));
1435 if (pv->eyes() == Eyes::BOTH || pv->eyes() == Eyes::RIGHT) {
1436 _next_video_time = time + one_video_frame();
1438 _next_video_eyes = increment_eyes (pv->eyes());
1440 if (_delay.size() < 3) {
1444 auto to_do = _delay.front();
1446 do_emit_video (to_do.first, to_do.second);
1451 Player::do_emit_video (shared_ptr<PlayerVideo> pv, DCPTime time)
1453 if (pv->eyes() == Eyes::BOTH || pv->eyes() == Eyes::RIGHT) {
1454 std::for_each(_active_texts.begin(), _active_texts.end(), [time](ActiveText& a) { a.clear_before(time); });
1457 auto subtitles = open_subtitles_for_frame (time);
1459 pv->set_text (subtitles.get ());
1467 Player::emit_audio (shared_ptr<AudioBuffers> data, DCPTime time)
1469 auto film = _film.lock();
1470 DCPOMATIC_ASSERT(film);
1472 /* Log if the assert below is about to fail */
1473 if (_next_audio_time && labs(time.get() - _next_audio_time->get()) > 1) {
1474 film->log()->log(String::compose("Out-of-sequence emit %1 vs %2", to_string(time), to_string(*_next_audio_time)), LogEntry::TYPE_WARNING);
1477 /* This audio must follow on from the previous, allowing for half a sample (at 48kHz) leeway */
1478 DCPOMATIC_ASSERT (!_next_audio_time || labs(time.get() - _next_audio_time->get()) < 2);
1479 Audio(data, time, film->audio_frame_rate());
1480 _next_audio_time = time + DCPTime::from_frames(data->frames(), film->audio_frame_rate());
1485 Player::fill_audio (DCPTimePeriod period)
1487 auto film = _film.lock();
1488 DCPOMATIC_ASSERT(film);
1490 if (period.from == period.to) {
1494 DCPOMATIC_ASSERT (period.from < period.to);
1496 DCPTime t = period.from;
1497 while (t < period.to) {
1498 DCPTime block = min (DCPTime::from_seconds (0.5), period.to - t);
1499 Frame const samples = block.frames_round(film->audio_frame_rate());
1501 auto silence = make_shared<AudioBuffers>(film->audio_channels(), samples);
1502 silence->make_silent ();
1503 emit_audio (silence, t);
1511 Player::one_video_frame () const
1513 auto film = _film.lock();
1514 DCPOMATIC_ASSERT(film);
1516 return DCPTime::from_frames(1, film->video_frame_rate ());
1520 pair<shared_ptr<AudioBuffers>, DCPTime>
1521 Player::discard_audio (shared_ptr<const AudioBuffers> audio, DCPTime time, DCPTime discard_to) const
1523 auto film = _film.lock();
1524 DCPOMATIC_ASSERT(film);
1526 auto const discard_time = discard_to - time;
1527 auto const discard_frames = discard_time.frames_round(film->audio_frame_rate());
1528 auto remaining_frames = audio->frames() - discard_frames;
1529 if (remaining_frames <= 0) {
1530 return make_pair(shared_ptr<AudioBuffers>(), DCPTime());
1532 auto cut = make_shared<AudioBuffers>(audio, remaining_frames, discard_frames);
1533 return make_pair(cut, time + discard_time);
1538 Player::set_dcp_decode_reduction (optional<int> reduction)
1540 ChangeSignaller<Player, int> cc(this, PlayerProperty::DCP_DECODE_REDUCTION);
1542 if (reduction == _dcp_decode_reduction.load()) {
1547 _dcp_decode_reduction = reduction;
1553 Player::content_time_to_dcp (shared_ptr<const Content> content, ContentTime t) const
1555 boost::mutex::scoped_lock lm (_mutex);
1557 for (auto i: _pieces) {
1558 if (i->content == content) {
1559 return content_time_to_dcp (i, t);
1563 /* We couldn't find this content; perhaps things are being changed over */
1568 optional<ContentTime>
1569 Player::dcp_to_content_time (shared_ptr<const Content> content, DCPTime t) const
1571 boost::mutex::scoped_lock lm (_mutex);
1573 for (auto i: _pieces) {
1574 if (i->content == content) {
1575 return dcp_to_content_time (i, t);
1579 /* We couldn't find this content; perhaps things are being changed over */
1584 shared_ptr<const Playlist>
1585 Player::playlist () const
1587 auto film = _film.lock();
1592 return _playlist ? _playlist : film->playlist();
1597 Player::atmos (weak_ptr<Piece> weak_piece, ContentAtmos data)
1603 auto film = _film.lock();
1604 DCPOMATIC_ASSERT(film);
1606 auto piece = weak_piece.lock ();
1607 DCPOMATIC_ASSERT (piece);
1609 auto const vfr = film->video_frame_rate();
1611 DCPTime const dcp_time = DCPTime::from_frames(data.frame, vfr) - DCPTime(piece->content->trim_start(), FrameRateChange(vfr, vfr));
1612 if (dcp_time < piece->content->position() || dcp_time >= (piece->content->end(film))) {
1616 auto ignore_atmos = std::find_if(
1617 piece->ignore_atmos.begin(),
1618 piece->ignore_atmos.end(),
1619 [dcp_time](DCPTimePeriod period) { return period.contains(dcp_time); }
1621 if (ignore_atmos != piece->ignore_atmos.end()) {
1625 Atmos (data.data, dcp_time, data.metadata);
1630 Player::signal_change(ChangeType type, int property)
1632 Change(type, property, false);