Various work on the audio code.

diff --git a/src/lib/audio_decoder.cc b/src/lib/audio_decoder.cc
index b866d3ecf14f65abfb99278fa6e35d6f0e9b25af..6a795f3ac1906c3bf0979c9623424ae36a588e41 100644 (file)
--- a/src/lib/audio_decoder.cc
+++ b/src/lib/audio_decoder.cc
@@ -94,11 +94,18 @@ AudioDecoder::set_fast ()
 optional<ContentTime>
 AudioDecoder::position () const
 {
-       optional<ContentTime> pos;
-       for (StreamMap::const_iterator i = _streams.begin(); i != _streams.end(); ++i) {
-               if (!pos || (i->second->position() && i->second->position().get() < pos.get())) {
-                       pos = i->second->position();
+       optional<ContentTime> p;
+       for (map<AudioStreamPtr, ContentTime>::const_iterator i = _positions.begin(); i != _positions.end(); ++i) {
+               if (!p || i->second < *p) {
+                       p = i->second;
                }
        }
-       return pos;
+
+       return p;
+}
+
+void
+AudioDecoder::set_position (AudioStreamPtr stream, ContentTime time)
+{
+       _positions[stream] = time;
 }

diff --git a/src/lib/audio_decoder.h b/src/lib/audio_decoder.h
index a777592c258f282b0e1dc73384f6174128a4881a..68e99608fb2b42c2631d718018a59ebd8af685f5 100644 (file)
--- a/src/lib/audio_decoder.h
+++ b/src/lib/audio_decoder.h
@@ -45,19 +45,22 @@ class AudioDecoder : public boost::enable_shared_from_this<AudioDecoder>, public
 public:
        AudioDecoder (Decoder* parent, boost::shared_ptr<const AudioContent>, boost::shared_ptr<Log> log);
 
+       boost::optional<ContentTime> position () const;
+       void set_position (AudioStreamPtr stream, ContentTime position);
+
        void set_fast ();
        void flush ();
 
        void emit (AudioStreamPtr stream, boost::shared_ptr<const AudioBuffers>, ContentTime);
 
-       boost::signals2::signal<void (ContentAudio)> Data;
+       boost::signals2::signal<void (AudioStreamPtr, ContentAudio)> Data;
 
        boost::optional<ContentTime> position () const;
 
 private:
        /** An AudioDecoderStream object to manage each stream in _audio_content */
-       typedef std::map<AudioStreamPtr, boost::shared_ptr<AudioDecoderStream> > StreamMap;
-       StreamMap _streams;
+       std::map<AudioStreamPtr, boost::shared_ptr<AudioDecoderStream> > _streams;
+       std::map<AudioStreamPtr, ContentTime> _positions;
 };
 
 #endif

diff --git a/src/lib/audio_merger.cc b/src/lib/audio_merger.cc
new file mode 100644 (file)
index 0000000..a0d300b
--- /dev/null
+++ b/src/lib/audio_merger.cc
@@ -0,0 +1,77 @@
+/*
+    Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+
+#include "audio_merger.h"
+#include "dcpomatic_time.h"
+
+using std::pair;
+using std::min;
+using std::max;
+using std::make_pair;
+using boost::shared_ptr;
+
+AudioMerger::AudioMerger (int channels, int frame_rate)
+       : _buffers (new AudioBuffers (channels, 0))
+       , _last_pull (0)
+       , _frame_rate (frame_rate)
+{
+
+}
+
+/** Pull audio up to a given time; after this call, no more data can be pushed
+ *  before the specified time.
+ */
+pair<shared_ptr<AudioBuffers>, DCPTime>
+AudioMerger::pull (DCPTime time)
+{
+       /* Number of frames to return */
+       Frame const to_return = time.frames_floor (_frame_rate) - _last_pull.frames_floor (_frame_rate);
+       shared_ptr<AudioBuffers> out (new AudioBuffers (_buffers->channels(), to_return));
+
+       /* And this is how many we will get from our buffer */
+       Frame const to_return_from_buffers = min (to_return, Frame (_buffers->frames()));
+
+       /* Copy the data that we have to the back end of the return buffer */
+       out->copy_from (_buffers.get(), to_return_from_buffers, 0, to_return - to_return_from_buffers);
+       /* Silence any gap at the start */
+       out->make_silent (0, to_return - to_return_from_buffers);
+
+       DCPTime out_time = _last_pull;
+       _last_pull = time;
+
+       /* And remove the data we're returning from our buffers */
+       if (_buffers->frames() > to_return_from_buffers) {
+               _buffers->move (to_return_from_buffers, 0, _buffers->frames() - to_return_from_buffers);
+       }
+       _buffers->set_frames (_buffers->frames() - to_return_from_buffers);
+
+       return make_pair (out, out_time);
+}
+
+void
+AudioMerger::push (boost::shared_ptr<const AudioBuffers> audio, DCPTime time)
+{
+       DCPOMATIC_ASSERT (time >= _last_pull);
+
+       Frame const frame = time.frames_floor (_frame_rate);
+       Frame after = max (Frame (_buffers->frames()), frame + audio->frames() - _last_pull.frames_floor (_frame_rate));
+       _buffers->ensure_size (after);
+       _buffers->accumulate_frames (audio.get(), 0, frame - _last_pull.frames_floor (_frame_rate), audio->frames ());
+       _buffers->set_frames (after);
+}

diff --git a/src/lib/audio_merger.h b/src/lib/audio_merger.h
new file mode 100644 (file)
index 0000000..c9026b9
--- /dev/null
+++ b/src/lib/audio_merger.h
@@ -0,0 +1,39 @@
+/*
+    Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+
+#include "audio_buffers.h"
+#include "dcpomatic_time.h"
+#include "util.h"
+
+class AudioMerger
+{
+public:
+       AudioMerger (int channels, int frame_rate);
+
+       /** Pull audio up to a given time; after this call, no more data can be pushed
+        *  before the specified time.
+        */
+       std::pair<boost::shared_ptr<AudioBuffers>, DCPTime> pull (DCPTime time);
+       void push (boost::shared_ptr<const AudioBuffers> audio, DCPTime time);
+
+private:
+       boost::shared_ptr<AudioBuffers> _buffers;
+       DCPTime _last_pull;
+       int _frame_rate;
+};

diff --git a/src/lib/dcp_decoder.cc b/src/lib/dcp_decoder.cc
index 25c805d3f2ab74d8a9c77c7d7941d18a8e7aad93..665b5e560ccdd949c5a67b4fc4e67509800fb5d6 100644 (file)
--- a/src/lib/dcp_decoder.cc
+++ b/src/lib/dcp_decoder.cc
@@ -155,6 +155,9 @@ DCPDecoder::pass ()
                }
        }
 
+       video->set_position (_next);
+       audio->set_position (_dcp_content->audio->stream(), _next);
+       subtitle->set_position (_next);
        _next += ContentTime::from_frames (1, vfr);
 
        if ((*_reel)->main_picture ()) {

diff --git a/src/lib/decoder_part.h b/src/lib/decoder_part.h
index 36594773a0dfa829d296de282268649ecddf81f6..39f77e6e6cbabedffc1e6bd538135a1db7c73019 100644 (file)
--- a/src/lib/decoder_part.h
+++ b/src/lib/decoder_part.h
@@ -33,6 +33,8 @@ public:
        DecoderPart (Decoder* parent, boost::shared_ptr<Log> log);
        virtual ~DecoderPart () {}
 
+       virtual boost::optional<ContentTime> position () const = 0;
+
        void set_ignore () {
                _ignore = true;
        }
@@ -41,8 +43,6 @@ public:
                return _ignore;
        }
 
-       virtual boost::optional<ContentTime> position () const = 0;
-
 protected:
        Decoder* _parent;
        boost::shared_ptr<Log> _log;

diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc
index 32903a20eb8995d4a2c2847f3b8b2d10e037afc5..57fe55892c3b4a85a06ab046c4bc46def689d99f 100644 (file)
--- a/src/lib/ffmpeg_decoder.cc
+++ b/src/lib/ffmpeg_decoder.cc
@@ -405,6 +405,8 @@ FFmpegDecoder::decode_audio_packet ()
                                LOG_WARNING ("Crazy timestamp %1", to_string (ct));
                        }
 
+                       audio->set_position (*stream, ct);
+
                        /* Give this data provided there is some, and its time is sane */
                        if (ct >= ContentTime() && data->frames() > 0) {
                                audio->emit (*stream, data, ct);

diff --git a/src/lib/player.cc b/src/lib/player.cc
index 924e60be219122251e50594039c5b4a57b75ea01..1e0a6dcf03b6044fc1dda191217cdf72d85834a1 100644 (file)
--- a/src/lib/player.cc
+++ b/src/lib/player.cc
@@ -103,6 +103,7 @@ Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist
        , _always_burn_subtitles (false)
        , _fast (false)
        , _play_referenced (false)
+       , _audio_merger (_film->audio_channels(), _film->audio_frame_rate())
 {
        _film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
        _playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
@@ -156,7 +157,7 @@ Player::setup_pieces ()
                }
 
                if (decoder->audio) {
-                       decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1));
+                       decoder->audio->Data.connect (bind (&Player::audio, this, weak_ptr<Piece> (piece), _1, _2));
                }
 
                if (decoder->subtitle) {
@@ -354,6 +355,14 @@ Player::dcp_to_resampled_audio (shared_ptr<const Piece> piece, DCPTime t) const
        return max (DCPTime (), DCPTime (piece->content->trim_start (), piece->frc) + s).frames_floor (_film->audio_frame_rate ());
 }
 
+DCPTime
+Player::resampled_audio_to_dcp (shared_ptr<const Piece> piece, Frame f) const
+{
+       /* See comment in dcp_to_content_video */
+       DCPTime const d = DCPTime::from_frames (f, _film->audio_frame_rate()) - DCPTime (piece->content->trim_start (), piece->frc);
+       return max (DCPTime (), d + piece->content->position ());
+}
+
 ContentTime
 Player::dcp_to_content_subtitle (shared_ptr<const Piece> piece, DCPTime t) const
 {
@@ -520,7 +529,11 @@ Player::pass ()
        shared_ptr<Piece> earliest;
        DCPTime earliest_position;
        BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
-               /* Convert i->decoder->position() to DCPTime and work out the earliest */
+               DCPTime const t = i->content->position() + DCPTime (i->decoder->position(), i->frc);
+               if (t < earliest_position) {
+                       earliest_position = t;
+                       earliest = i;
+               }
        }
 
        earliest->decoder->pass ();
@@ -536,58 +549,112 @@ Player::video (weak_ptr<Piece> wp, ContentVideo video)
                return;
        }
 
-       /* Get subs to burn in and burn them in */
+       /* XXX: get subs to burn in and burn them in */
 
 
        /* Fill gaps */
 
-       DCPTime time = content_video_to_dcp (piece, video.frame.index());
+       DCPTime const time = content_video_to_dcp (piece, video.frame.index());
 
-       dcp::Size image_size = piece->content->video->scale().size (
-               piece->content->video, _video_container_size, _film->frame_size ()
-               );
+       for (DCPTime i = _last_video_time; i < time; i += DCPTime::from_frames (1, _film->video_frame_rate())) {
+               if (_playlist->video_content_at(i) && _last_video) {
+                       Video (_last_video->clone (i));
+               } else {
+                       Video (black_player_video_frame (i));
+               }
+       }
 
-       Video (
-               shared_ptr<PlayerVideo> (
-                       new PlayerVideo (
-                               video.image,
-                               time,
-                               piece->content->video->crop (),
-                               piece->content->video->fade (video.frame.index()),
-                               image_size,
-                               _video_container_size,
-                               video.frame.eyes(),
-                               video.part,
-                               piece->content->video->colour_conversion ()
-                               )
+       _last_video.reset (
+               new PlayerVideo (
+                       video.image,
+                       time,
+                       piece->content->video->crop (),
+                       piece->content->video->fade (video.frame.index()),
+                       piece->content->video->scale().size (
+                               piece->content->video, _video_container_size, _film->frame_size ()
+                               ),
+                       _video_container_size,
+                       video.frame.eyes(),
+                       video.part,
+                       piece->content->video->colour_conversion ()
                        )
                );
 
+       _last_video_time = time;
+
+       Video (_last_video);
 }
 
 void
-Player::audio (weak_ptr<Piece> piece, ContentAudio video)
+Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_audio)
 {
-       /* gain, remap, processor */
-       /* Put into merge buffer */
+       shared_ptr<Piece> piece = wp.lock ();
+       if (!piece) {
+               return;
+       }
+
+       shared_ptr<AudioContent> content = piece->content->audio;
+       DCPOMATIC_ASSERT (content);
+
+       shared_ptr<AudioBuffers> audio = content_audio.audio;
+
+       /* Gain */
+       if (content->gain() != 0) {
+               shared_ptr<AudioBuffers> gain (new AudioBuffers (audio));
+               gain->apply_gain (content->gain ());
+               audio = gain;
+       }
+
+       /* XXX: end-trimming used to be checked here */
+
+       /* Compute time in the DCP */
+       DCPTime const time = resampled_audio_to_dcp (piece, content_audio.frame) + DCPTime::from_seconds (content->delay() / 1000);
+
+       /* Remap channels */
+       shared_ptr<AudioBuffers> dcp_mapped (new AudioBuffers (_film->audio_channels(), audio->frames()));
+       dcp_mapped->make_silent ();
+
+       AudioMapping map = stream->mapping ();
+       for (int i = 0; i < map.input_channels(); ++i) {
+               for (int j = 0; j < dcp_mapped->channels(); ++j) {
+                       if (map.get (i, static_cast<dcp::Channel> (j)) > 0) {
+                               dcp_mapped->accumulate_channel (
+                                       audio.get(),
+                                       i,
+                                       static_cast<dcp::Channel> (j),
+                                       map.get (i, static_cast<dcp::Channel> (j))
+                                       );
+                       }
+               }
+       }
+
+       audio = dcp_mapped;
+
+       if (_audio_processor) {
+               audio = _audio_processor->run (audio, _film->audio_channels ());
+       }
+
+       _audio_merger.push (audio, time);
 }
 
 void
 Player::image_subtitle (weak_ptr<Piece> piece, ContentImageSubtitle subtitle)
 {
-       /* Store for video to see */
+       /* XXX: Store for video to see */
 }
 
 void
 Player::text_subtitle (weak_ptr<Piece> piece, ContentTextSubtitle subtitle)
 {
-       /* Store for video to see, or emit */
+       /* XXX: Store for video to see, or emit */
 }
 
 void
 Player::seek (DCPTime time, bool accurate)
 {
+       /* XXX: seek decoders */
+
        if (accurate) {
-               _last_video = time - DCPTime::from_frames (1, _film->video_frame_rate ());
+               _last_video_time = time - DCPTime::from_frames (1, _film->video_frame_rate ());
        }
 }

diff --git a/src/lib/player.h b/src/lib/player.h
index c0e0f9f70f3aa0daa996e4224354f3d06b45b018..d0d68c064a3eaf4198c785ae8b12e1dc667c37b6 100644 (file)
--- a/src/lib/player.h
+++ b/src/lib/player.h
@@ -29,6 +29,8 @@
 #include "content_video.h"
 #include "content_audio.h"
 #include "content_subtitle.h"
+#include "audio_stream.h"
+#include "audio_merger.h"
 #include <boost/shared_ptr.hpp>
 #include <boost/enable_shared_from_this.hpp>
 #include <list>
@@ -95,12 +97,13 @@ private:
        Frame dcp_to_content_video (boost::shared_ptr<const Piece> piece, DCPTime t) const;
        DCPTime content_video_to_dcp (boost::shared_ptr<const Piece> piece, Frame f) const;
        Frame dcp_to_resampled_audio (boost::shared_ptr<const Piece> piece, DCPTime t) const;
+       DCPTime resampled_audio_to_dcp (boost::shared_ptr<const Piece> piece, Frame f) const;
        ContentTime dcp_to_content_subtitle (boost::shared_ptr<const Piece> piece, DCPTime t) const;
        DCPTime content_subtitle_to_dcp (boost::shared_ptr<const Piece> piece, ContentTime t) const;
        boost::shared_ptr<PlayerVideo> black_player_video_frame (DCPTime) const;
        std::list<boost::shared_ptr<Piece> > overlaps (DCPTime from, DCPTime to, boost::function<bool (Content *)> valid);
        void video (boost::weak_ptr<Piece>, ContentVideo);
-       void audio (boost::weak_ptr<Piece>, ContentAudio);
+       void audio (boost::weak_ptr<Piece>, AudioStreamPtr, ContentAudio);
        void image_subtitle (boost::weak_ptr<Piece>, ContentImageSubtitle);
        void text_subtitle (boost::weak_ptr<Piece>, ContentTextSubtitle);
 
@@ -128,7 +131,10 @@ private:
        /** true if we should `play' (i.e output) referenced DCP data (e.g. for preview) */
        bool _play_referenced;
 
-       DCPTime _last_video;
+       boost::shared_ptr<PlayerVideo> _last_video;
+       DCPTime _last_video_time;
+
+       AudioMerger _audio_merger;
 
        boost::shared_ptr<AudioProcessor> _audio_processor;
 

diff --git a/src/lib/player_video.cc b/src/lib/player_video.cc
index 5e34ddf97fa0879868fb92d5e84e7cf5156bc607..7c9a4ee66fdeb5c08f747673df81c355d1e64eac 100644 (file)
--- a/src/lib/player_video.cc
+++ b/src/lib/player_video.cc
@@ -250,3 +250,11 @@ PlayerVideo::keep_xyz_or_rgb (AVPixelFormat p)
 {
        return p == AV_PIX_FMT_XYZ12LE ? AV_PIX_FMT_XYZ12LE : AV_PIX_FMT_RGB48LE;
 }
+
+shared_ptr<PlayerVideo>
+PlayerVideo::clone (DCPTime time) const
+{
+       shared_ptr<PlayerVideo> c (new PlayerVideo (*this));
+       c->_time = time;
+       return c;
+}

diff --git a/src/lib/player_video.h b/src/lib/player_video.h
index b5acd9686ef1037ce80698e6cd24a1c0cb0f519e..a5ca9b865705648bc298db707166d3614968873a 100644 (file)
--- a/src/lib/player_video.h
+++ b/src/lib/player_video.h
@@ -92,6 +92,8 @@ public:
 
        bool same (boost::shared_ptr<const PlayerVideo> other) const;
 
+       boost::shared_ptr<PlayerVideo> clone (DCPTime time) const;
+
 private:
        boost::shared_ptr<const ImageProxy> _in;
        DCPTime _time;

diff --git a/src/lib/playlist.cc b/src/lib/playlist.cc
index a30dde633cde3120bc29cdc2b762e3c723f802de..6abbbcab33810696dc2bbb551aa5cd512ed5ff98 100644 (file)
--- a/src/lib/playlist.cc
+++ b/src/lib/playlist.cc
@@ -536,3 +536,15 @@ Playlist::content_summary (DCPTimePeriod period) const
 
        return best_summary;
 }
+
+bool
+Playlist::video_content_at (DCPTime time) const
+{
+       BOOST_FOREACH (shared_ptr<Content> i, _content) {
+               if (i->video && i->position() <= time && time < i->end()) {
+                       return true;
+               }
+       }
+
+       return false;
+}

diff --git a/src/lib/playlist.h b/src/lib/playlist.h
index 0a5c087dea67ed076f0833e584ef2c5b3359a823..335e26013b02f1ad3eb51e4fefafd659acdc9924 100644 (file)
--- a/src/lib/playlist.h
+++ b/src/lib/playlist.h
@@ -55,6 +55,7 @@ public:
        void move_later (boost::shared_ptr<Content>);
 
        ContentList content () const;
+       bool video_content_at (DCPTime time) const;
 
        std::string video_identifier () const;
 

diff --git a/src/lib/subtitle_decoder.h b/src/lib/subtitle_decoder.h
index 904aaed7823a4e778d10af9be8e442b93f80a83a..a1c4b3ddc34b5db80455ec694243df81848e7dae 100644 (file)
--- a/src/lib/subtitle_decoder.h
+++ b/src/lib/subtitle_decoder.h
@@ -48,6 +48,14 @@ public:
                boost::shared_ptr<Log> log
                );
 
+       void set_position (ContentTime position) {
+               _position = position;
+       }
+
+       boost::optional<ContentTime> position () const {
+               return _position;
+       }
+
        void emit_image (ContentTimePeriod period, boost::shared_ptr<Image>, dcpomatic::Rect<double>);
        void emit_text (ContentTimePeriod period, std::list<dcp::SubtitleString>);
        void emit_text (ContentTimePeriod period, sub::Subtitle const & subtitle);
@@ -61,6 +69,7 @@ public:
 
 private:
        boost::shared_ptr<const SubtitleContent> _content;
+       boost::optional<ContentTime> _position;
 };
 
 #endif

diff --git a/src/lib/video_decoder.h b/src/lib/video_decoder.h
index 08173d34d3fd9655a2c69faa07cbc15613ba2c08..e9bef24f81d61cd184ddfd3e81e83eeb282b796a 100644 (file)
--- a/src/lib/video_decoder.h
+++ b/src/lib/video_decoder.h
@@ -51,6 +51,14 @@ public:
        friend struct ffmpeg_pts_offset_test;
        friend void ffmpeg_decoder_sequential_test_one (boost::filesystem::path file, float fps, int gaps, int video_length);
 
+       void set_position (ContentTime position) {
+               _position = position;
+       }
+
+       boost::optional<ContentTime> position () const {
+               return _position;
+       }
+
        void emit (boost::shared_ptr<const ImageProxy>, Frame frame);
 
        boost::signals2::signal<void (ContentVideo)> Data;
@@ -58,6 +66,7 @@ public:
 private:
        boost::shared_ptr<const Content> _content;
        boost::optional<Frame> _last_emitted;
+       boost::optional<ContentTime> _position;
 };
 
 #endif

diff --git a/src/lib/wscript b/src/lib/wscript
index 14e14676ec0da8fdd7c55fb7439e7388f3ca5a34..0e4e5bcfc34371cfa278af370c356af71e9ea435 100644 (file)
--- a/src/lib/wscript
+++ b/src/lib/wscript
@@ -32,6 +32,7 @@ sources = """
           audio_filter.cc
           audio_filter_graph.cc
           audio_mapping.cc
+          audio_merger.cc
           audio_point.cc
           audio_processor.cc
           audio_stream.cc