Fix merging of audio in various circumstances.

14 files changed, 274 insertions, 101 deletions

diff --git a/src/lib/audio_buffers.cc b/src/lib/audio_buffers.cc
index 546abbb54..f01f8baaf 100644
--- a/src/lib/audio_buffers.cc
+++ b/src/lib/audio_buffers.cc
@@ -200,7 +200,7 @@ AudioBuffers::copy_from (AudioBuffers const * from, int32_t frames_to_copy, int3
  */
 
 void
-AudioBuffers::move (int32_t from, int32_t to, int32_t frames)
+AudioBuffers::move (int32_t frames, int32_t from, int32_t to)
 {
 	if (frames == 0) {
 		return;
@@ -273,7 +273,7 @@ AudioBuffers::ensure_size (int32_t frames)
 }
 
 void
-AudioBuffers::accumulate_frames (AudioBuffers const * from, int32_t read_offset, int32_t write_offset, int32_t frames)
+AudioBuffers::accumulate_frames (AudioBuffers const * from, int32_t frames, int32_t read_offset, int32_t write_offset)
 {
 	DCPOMATIC_ASSERT (_channels == from->channels ());
 	DCPOMATIC_ASSERT (read_offset >= 0);
@@ -325,3 +325,19 @@ AudioBuffers::clone () const
 	b->copy_from (this, frames (), 0, 0);
 	return b;
 }
+
+void
+AudioBuffers::append (shared_ptr<const AudioBuffers> other)
+{
+	ensure_size (_frames + other->frames());
+	copy_from (other.get(), other->frames(), 0, _frames);
+	_frames += other->frames();
+}
+
+void
+AudioBuffers::trim_start (int32_t frames)
+{
+	DCPOMATIC_ASSERT (frames <= _frames);
+	move (_frames - frames, frames, 0);
+	set_frames (_frames - frames);
+}
diff --git a/src/lib/audio_buffers.h b/src/lib/audio_buffers.h
index a294ff914..991ef5334 100644
--- a/src/lib/audio_buffers.h
+++ b/src/lib/audio_buffers.h
@@ -74,9 +74,11 @@ public:
 
 	void copy_from (AudioBuffers const * from, int32_t frames_to_copy, int32_t read_offset, int32_t write_offset);
 	void copy_channel_from (AudioBuffers const * from, int from_channel, int to_channel);
-	void move (int32_t from, int32_t to, int32_t frames);
+	void move (int32_t frames, int32_t from, int32_t to);
 	void accumulate_channel (AudioBuffers const * from, int from_channel, int to_channel, float gain = 1);
-	void accumulate_frames (AudioBuffers const *, int32_t read_offset, int32_t write_offset, int32_t frames);
+	void accumulate_frames (AudioBuffers const * from, int32_t frames, int32_t read_offset, int32_t write_offset);
+	void append (boost::shared_ptr<const AudioBuffers> other);
+	void trim_start (int32_t frames);
 
 private:
 	void allocate (int channels, int32_t frames);
diff --git a/src/lib/audio_delay.cc b/src/lib/audio_delay.cc
index 893773ddd..391a201a2 100644
--- a/src/lib/audio_delay.cc
+++ b/src/lib/audio_delay.cc
@@ -71,7 +71,7 @@ AudioDelay::run (shared_ptr<const AudioBuffers> in)
 		}
 
 		/* Shuffle the tail down */
-		_tail->move (out->frames(), 0, _tail->frames() - out->frames());
+		_tail->move (_tail->frames() - out->frames(), out->frames(), 0);
 
 		/* Copy input into the tail */
 		_tail->copy_from (in.get(), in->frames(), 0, _tail->frames() - in->frames());
diff --git a/src/lib/audio_filter.cc b/src/lib/audio_filter.cc
index 44345fc9d..b3916d9bd 100644
--- a/src/lib/audio_filter.cc
+++ b/src/lib/audio_filter.cc
@@ -105,7 +105,7 @@ AudioFilter::run (shared_ptr<const AudioBuffers> in)
 
 	int const amount = min (in->frames(), _tail->frames());
 	if (amount < _tail->frames ()) {
-		_tail->move (amount, 0, _tail->frames() - amount);
+		_tail->move (_tail->frames() - amount, amount, 0);
 	}
 	_tail->copy_from (in.get(), amount, in->frames() - amount, _tail->frames () - amount);
 
diff --git a/src/lib/audio_merger.cc b/src/lib/audio_merger.cc
index 49cdea6a3..5e0589bb1 100644
--- a/src/lib/audio_merger.cc
+++ b/src/lib/audio_merger.cc
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2013-2017 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -24,12 +24,14 @@
 using std::pair;
 using std::min;
 using std::max;
+using std::list;
+using std::cout;
 using std::make_pair;
 using boost::shared_ptr;
+using boost::optional;
 
-AudioMerger::AudioMerger (int channels, int frame_rate)
-	: _buffers (new AudioBuffers (channels, 0))
-	, _last_pull (0)
+AudioMerger::AudioMerger (int frame_rate)
+	: _last_pull (0)
 	, _frame_rate (frame_rate)
 {
 
@@ -38,31 +40,36 @@ AudioMerger::AudioMerger (int channels, int frame_rate)
 /** Pull audio up to a given time; after this call, no more data can be pushed
  *  before the specified time.
  */
-pair<shared_ptr<AudioBuffers>, DCPTime>
+list<pair<shared_ptr<AudioBuffers>, DCPTime> >
 AudioMerger::pull (DCPTime time)
 {
-	/* Number of frames to return */
-	Frame const to_return = time.frames_floor (_frame_rate) - _last_pull.frames_floor (_frame_rate);
-	shared_ptr<AudioBuffers> out (new AudioBuffers (_buffers->channels(), to_return));
-
-	/* And this is how many we will get from our buffer */
-	Frame const to_return_from_buffers = min (to_return, Frame (_buffers->frames()));
-
-	/* Copy the data that we have to the back end of the return buffer */
-	out->copy_from (_buffers.get(), to_return_from_buffers, 0, to_return - to_return_from_buffers);
-	/* Silence any gap at the start */
-	out->make_silent (0, to_return - to_return_from_buffers);
-
-	DCPTime out_time = _last_pull;
-	_last_pull = time;
-
-	/* And remove the data we're returning from our buffers */
-	if (_buffers->frames() > to_return_from_buffers) {
-		_buffers->move (to_return_from_buffers, 0, _buffers->frames() - to_return_from_buffers);
+	list<pair<shared_ptr<AudioBuffers>, DCPTime> > out;
+
+	DCPTimePeriod period (_last_pull, time);
+	_buffers.sort (AudioMerger::BufferComparator());
+
+	list<Buffer> new_buffers;
+
+	BOOST_FOREACH (Buffer i, _buffers) {
+		if (i.period().to < time) {
+			/* Completely within the pull period */
+			out.push_back (make_pair (i.audio, i.time));
+		} else if (i.time < time) {
+			/* Overlaps the end of the pull period */
+			shared_ptr<AudioBuffers> audio (new AudioBuffers (i.audio->channels(), DCPTime(time - i.time).frames_floor(_frame_rate)));
+			audio->copy_from (i.audio.get(), audio->frames(), 0, 0);
+			out.push_back (make_pair (audio, i.time));
+			i.audio->trim_start (audio->frames ());
+			i.time += DCPTime::from_frames(audio->frames(), _frame_rate);
+			new_buffers.push_back (i);
+		} else {
+			/* Not involved */
+			new_buffers.push_back (i);
+		}
 	}
-	_buffers->set_frames (_buffers->frames() - to_return_from_buffers);
 
-	return make_pair (out, out_time);
+	_buffers = new_buffers;
+	return out;
 }
 
 void
@@ -70,9 +77,60 @@ AudioMerger::push (boost::shared_ptr<const AudioBuffers> audio, DCPTime time)
 {
 	DCPOMATIC_ASSERT (time >= _last_pull);
 
-	Frame const frame = time.frames_floor (_frame_rate);
-	Frame after = max (Frame (_buffers->frames()), frame + audio->frames() - _last_pull.frames_floor (_frame_rate));
-	_buffers->ensure_size (after);
-	_buffers->accumulate_frames (audio.get(), 0, frame - _last_pull.frames_floor (_frame_rate), audio->frames ());
-	_buffers->set_frames (after);
+	DCPTimePeriod period (time, time + DCPTime::from_frames (audio->frames(), _frame_rate));
+
+	/* Mix any parts of this new block with existing ones */
+	BOOST_FOREACH (Buffer i, _buffers) {
+		optional<DCPTimePeriod> overlap = i.period().overlap (period);
+		if (overlap) {
+			int32_t const offset = DCPTime(overlap->from - i.time).frames_floor(_frame_rate);
+			int32_t const frames = overlap->duration().frames_floor(_frame_rate);
+			if (i.time < time) {
+				i.audio->accumulate_frames(audio.get(), frames, 0, offset);
+			} else {
+				i.audio->accumulate_frames(audio.get(), frames, offset, 0);
+			}
+		}
+	}
+
+	list<DCPTimePeriod> periods;
+	BOOST_FOREACH (Buffer i, _buffers) {
+		periods.push_back (i.period ());
+	}
+
+	/* Add the non-overlapping parts */
+	BOOST_FOREACH (DCPTimePeriod i, subtract (period, periods)) {
+		list<Buffer>::iterator before = _buffers.end();
+		list<Buffer>::iterator after = _buffers.end();
+		for (list<Buffer>::iterator j = _buffers.begin(); j != _buffers.end(); ++j) {
+			if (j->period().to == i.from) {
+				before = j;
+			}
+			if (j->period().from == i.to) {
+				after = j;
+			}
+		}
+
+		/* Get the part of audio that we want to use */
+		shared_ptr<AudioBuffers> part (new AudioBuffers (audio->channels(), i.to.frames_floor(_frame_rate) - i.from.frames_floor(_frame_rate)));
+		part->copy_from (audio.get(), part->frames(), DCPTime(i.from - time).frames_floor(_frame_rate), 0);
+
+		if (before == _buffers.end() && after == _buffers.end()) {
+			/* New buffer */
+			_buffers.push_back (Buffer (part, time, _frame_rate));
+		} else if (before != _buffers.end() && after == _buffers.end()) {
+			/* We have an existing buffer before this one; append new data to it */
+			before->audio->append (part);
+		} else if (before ==_buffers.end() && after != _buffers.end()) {
+			/* We have an existing buffer after this one; append it to the new data and replace */
+			part->append (after->audio);
+			after->audio = part;
+			after->time = time;
+		} else {
+			/* We have existing buffers both before and after; coalesce them all */
+			before->audio->append (part);
+			before->audio->append (after->audio);
+			_buffers.erase (after);
+		}
+	}
 }
diff --git a/src/lib/audio_merger.h b/src/lib/audio_merger.h
index 6db28b6c3..87bda7f8b 100644
--- a/src/lib/audio_merger.h
+++ b/src/lib/audio_merger.h
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2013-2016 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2013-2017 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -24,19 +24,54 @@
 class AudioMerger
 {
 public:
-	AudioMerger (int channels, int frame_rate);
+	AudioMerger (int frame_rate);
 
 	/** Pull audio up to a given time; after this call, no more data can be pushed
 	 *  before the specified time.
 	 */
-	std::pair<boost::shared_ptr<AudioBuffers>, DCPTime> pull (DCPTime time);
+	std::list<std::pair<boost::shared_ptr<AudioBuffers>, DCPTime> > pull (DCPTime time);
 	void push (boost::shared_ptr<const AudioBuffers> audio, DCPTime time);
-	DCPTime last_pull () const {
-		return _last_pull;
-	}
 
 private:
-	boost::shared_ptr<AudioBuffers> _buffers;
+	class Buffer
+	{
+	public:
+		/** @param c Channels
+		 *  @param f Frames
+		 *  @param t Time
+		 *  @param r Frame rate.
+		 */
+		Buffer (int c, int32_t f, DCPTime t, int r)
+			: audio (new AudioBuffers (c, f))
+			, time (t)
+			, frame_rate (r)
+		{}
+
+		Buffer (boost::shared_ptr<AudioBuffers> a, DCPTime t, int r)
+			: audio (a)
+			, time (t)
+			, frame_rate (r)
+		{}
+
+		boost::shared_ptr<AudioBuffers> audio;
+		DCPTime time;
+		int frame_rate;
+
+		DCPTimePeriod period () const {
+			return DCPTimePeriod (time, time + DCPTime::from_frames (audio->frames(), frame_rate));
+		}
+	};
+
+	class BufferComparator
+	{
+	public:
+		bool operator() (AudioMerger::Buffer const & a, AudioMerger::Buffer const & b)
+		{
+			return a.time < b.time;
+		}
+	};
+
+	std::list<Buffer> _buffers;
 	DCPTime _last_pull;
 	int _frame_rate;
 };
diff --git a/src/lib/dcpomatic_time.h b/src/lib/dcpomatic_time.h
index 35ddd0199..cc31755cb 100644
--- a/src/lib/dcpomatic_time.h
+++ b/src/lib/dcpomatic_time.h
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2014-2016 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2014-2017 Carl Hetherington <cth@carlh.net>
 
     This file is part of DCP-o-matic.
 
@@ -28,6 +28,7 @@
 #include "frame_rate_change.h"
 #include "dcpomatic_assert.h"
 #include <boost/optional.hpp>
+#include <boost/foreach.hpp>
 #include <stdint.h>
 #include <cmath>
 #include <ostream>
@@ -291,6 +292,45 @@ public:
 	}
 };
 
+/** @param B Periods to subtract from `A', must be in ascending order of start time and must not overlap */
+template <class T>
+std::list<TimePeriod<T> > subtract (TimePeriod<T> A, std::list<TimePeriod<T> > const & B)
+{
+	std::list<TimePeriod<T> > result;
+	result.push_back (A);
+
+	BOOST_FOREACH (TimePeriod<T> i, B) {
+		std::list<TimePeriod<T> > new_result;
+		BOOST_FOREACH (TimePeriod<T> j, result) {
+			boost::optional<TimePeriod<T> > ov = i.overlap (j);
+			if (ov) {
+				if (*ov == i) {
+					/* A contains all of B */
+					if (i.from != j.from) {
+						new_result.push_back (TimePeriod<T> (j.from, i.from));
+					}
+					if (i.to != j.to) {
+						new_result.push_back (TimePeriod<T> (i.to, j.to));
+					}
+				} else if (*ov == j) {
+					/* B contains all of A */
+				} else if (i.from < j.from) {
+					/* B overlaps start of A */
+					new_result.push_back (TimePeriod<T> (i.to, j.to));
+				} else if (i.to > j.to) {
+					/* B overlaps end of A */
+					new_result.push_back (TimePeriod<T> (j.from, i.from));
+				}
+			} else {
+				new_result.push_back (j);
+			}
+		}
+		result = new_result;
+	}
+
+	return result;
+}
+
 typedef TimePeriod<ContentTime> ContentTimePeriod;
 typedef TimePeriod<DCPTime> DCPTimePeriod;
 
diff --git a/src/lib/decoder.cc b/src/lib/decoder.cc
index ee03a1579..fef5e2a99 100644
--- a/src/lib/decoder.cc
+++ b/src/lib/decoder.cc
@@ -49,7 +49,7 @@ Decoder::position () const
 }
 
 void
-Decoder::seek (ContentTime time, bool accurate)
+Decoder::seek (ContentTime, bool)
 {
 	if (audio) {
 		audio->seek ();
diff --git a/src/lib/decoder.h b/src/lib/decoder.h
index d87ff610a..8b901f70c 100644
--- a/src/lib/decoder.h
+++ b/src/lib/decoder.h
@@ -53,7 +53,7 @@ public:
 	virtual bool pass () = 0;
 	virtual void seek (ContentTime time, bool accurate);
 
-	ContentTime position () const;
+	virtual ContentTime position () const;
 };
 
 #endif
diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc
index 1bae99d63..604864a14 100644
--- a/src/lib/ffmpeg_decoder.cc
+++ b/src/lib/ffmpeg_decoder.cc
@@ -398,7 +398,7 @@ FFmpegDecoder::decode_audio_packet ()
 			if (ct < ContentTime ()) {
 				/* Discard audio data that comes before time 0 */
 				Frame const remove = min (int64_t (data->frames()), (-ct).frames_ceil(double((*stream)->frame_rate ())));
-				data->move (remove, 0, data->frames() - remove);
+				data->move (data->frames() - remove, remove, 0);
 				data->set_frames (data->frames() - remove);
 				ct += ContentTime::from_frames (remove, (*stream)->frame_rate ());
 			}
diff --git a/src/lib/player.cc b/src/lib/player.cc
index 7d53b07bf..c14b55be0 100644
--- a/src/lib/player.cc
+++ b/src/lib/player.cc
@@ -87,7 +87,7 @@ Player::Player (shared_ptr<const Film> film, shared_ptr<const Playlist> playlist
 	, _always_burn_subtitles (false)
 	, _fast (false)
 	, _play_referenced (false)
-	, _audio_merger (_film->audio_channels(), _film->audio_frame_rate())
+	, _audio_merger (_film->audio_frame_rate())
 {
 	_film_changed_connection = _film->Changed.connect (bind (&Player::film_changed, this, _1));
 	_playlist_changed_connection = _playlist->Changed.connect (bind (&Player::playlist_changed, this));
@@ -156,6 +156,20 @@ Player::setup_pieces ()
 		}
 	}
 
+	if (!_play_referenced) {
+		BOOST_FOREACH (shared_ptr<Piece> i, _pieces) {
+			shared_ptr<DCPContent> dc = dynamic_pointer_cast<DCPContent> (i->content);
+			if (dc) {
+				if (dc->reference_video()) {
+					_no_video.push_back (DCPTimePeriod (dc->position(), dc->end()));
+				}
+				if (dc->reference_audio()) {
+					_no_audio.push_back (DCPTimePeriod (dc->position(), dc->end()));
+				}
+			}
+		}
+	}
+
 	_have_valid_pieces = true;
 }
 
@@ -529,31 +543,13 @@ Player::pass ()
 	}
 
 	if (!earliest) {
-		/* No more content; fill up to the length of our playlist with silent black */
-
-		DCPTime const length = _playlist->length ();
-
-		DCPTime const frame = DCPTime::from_frames (1, _film->video_frame_rate());
-		DCPTime from;
+		/* No more content; fill up with silent black */
+		DCPTimePeriod remaining_video (DCPTime(), _playlist->length());
 		if (_last_time) {
-			from = _last_time.get() + frame;
-		}
-		for (DCPTime i = from; i < length; i += frame) {
-			Video (black_player_video_frame (), i);
-		}
-
-		DCPTime t = _last_audio_time;
-		while (t < length) {
-			DCPTime block = min (DCPTime::from_seconds (0.5), length - t);
-			Frame const samples = block.frames_round(_film->audio_frame_rate());
-			if (samples) {
-				shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
-				silence->make_silent ();
-				Audio (silence, t);
-			}
-			t += block;
+			remaining_video.from = _last_time.get() + one_video_frame();
 		}
-
+		fill_video (remaining_video);
+		fill_audio (DCPTimePeriod (_last_audio_time, _playlist->length()));
 		return true;
 	}
 
@@ -568,22 +564,12 @@ Player::pass ()
 		}
 	}
 
-//	cout << "PULL " << to_string(pull_from) << "\n";
-	pair<shared_ptr<AudioBuffers>, DCPTime> audio = _audio_merger.pull (pull_from);
-	if (audio.first->frames() > 0) {
-		DCPOMATIC_ASSERT (audio.second >= _last_audio_time);
-		DCPTime t = _last_audio_time;
-		while (t < audio.second) {
-			/* Silence up to the time of this new audio */
-			DCPTime block = min (DCPTime::from_seconds (0.5), audio.second - t);
-			shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), block.frames_round(_film->audio_frame_rate())));
-			silence->make_silent ();
-			Audio (silence, t);
-			t += block;
-		}
-
-		Audio (audio.first, audio.second);
-		_last_audio_time = audio.second + DCPTime::from_frames(audio.first->frames(), _film->audio_frame_rate());
+	list<pair<shared_ptr<AudioBuffers>, DCPTime> > audio = _audio_merger.pull (pull_from);
+	for (list<pair<shared_ptr<AudioBuffers>, DCPTime> >::iterator i = audio.begin(); i != audio.end(); ++i) {
+		DCPOMATIC_ASSERT (i->second >= _last_audio_time);
+		fill_audio (DCPTimePeriod (_last_audio_time, i->second));
+		Audio (i->first, i->second);
+		_last_audio_time = i->second + DCPTime::from_frames(i->first->frames(), _film->audio_frame_rate());
 	}
 
 	return false;
@@ -604,7 +590,7 @@ Player::video (weak_ptr<Piece> wp, ContentVideo video)
 
 	/* Time and period of the frame we will emit */
 	DCPTime const time = content_video_to_dcp (piece, video.frame);
-	DCPTimePeriod const period (time, time + DCPTime::from_frames (1, _film->video_frame_rate()));
+	DCPTimePeriod const period (time, time + one_video_frame());
 
 	/* Discard if it's outside the content's period */
 	if (time < piece->content->position() || time >= piece->content->end()) {
@@ -641,15 +627,7 @@ Player::video (weak_ptr<Piece> wp, ContentVideo video)
 	/* Fill gaps */
 
 	if (_last_time) {
-		/* XXX: this may not work for 3D */
-		DCPTime const frame = DCPTime::from_frames (1, _film->video_frame_rate());
-		for (DCPTime i = _last_time.get() + frame; i < time; i += frame) {
-			if (_playlist->video_content_at(i) && _last_video) {
-				Video (shared_ptr<PlayerVideo> (new PlayerVideo (*_last_video)), i);
-			} else {
-				Video (black_player_video_frame (), i);
-			}
-		}
+		fill_video (DCPTimePeriod (_last_time.get() + one_video_frame(), time));
 	}
 
 	_last_video.reset (
@@ -759,7 +737,6 @@ Player::audio (weak_ptr<Piece> wp, AudioStreamPtr stream, ContentAudio content_a
 		content_audio.audio = _audio_processor->run (content_audio.audio, _film->audio_channels ());
 	}
 
-//	cout << "PUSH " << content_audio.audio->frames() << " @ " << to_string(time) << "\n";
 	_audio_merger.push (content_audio.audio, time);
 
 	DCPOMATIC_ASSERT (_stream_states.find (stream) != _stream_states.end ());
@@ -852,7 +829,7 @@ Player::seek (DCPTime time, bool accurate)
 	}
 
 	if (accurate) {
-		_last_time = time - DCPTime::from_frames (1, _film->video_frame_rate ());
+		_last_time = time - one_video_frame ();
 	} else {
 		_last_time = optional<DCPTime> ();
 	}
@@ -884,3 +861,42 @@ Player::resampler (shared_ptr<const AudioContent> content, AudioStreamPtr stream
 	_resamplers[make_pair(content, stream)] = r;
 	return r;
 }
+
+void
+Player::fill_video (DCPTimePeriod period)
+{
+	/* XXX: this may not work for 3D */
+	BOOST_FOREACH (DCPTimePeriod i, subtract(period, _no_video)) {
+		for (DCPTime j = i.from; j < i.to; j += one_video_frame()) {
+			if (_playlist->video_content_at(j) && _last_video) {
+				Video (shared_ptr<PlayerVideo> (new PlayerVideo (*_last_video)), j);
+			} else {
+				Video (black_player_video_frame(), j);
+			}
+		}
+	}
+}
+
+void
+Player::fill_audio (DCPTimePeriod period)
+{
+	BOOST_FOREACH (DCPTimePeriod i, subtract(period, _no_audio)) {
+		DCPTime t = i.from;
+		while (t < i.to) {
+			DCPTime block = min (DCPTime::from_seconds (0.5), i.to - t);
+			Frame const samples = block.frames_round(_film->audio_frame_rate());
+			if (samples) {
+				shared_ptr<AudioBuffers> silence (new AudioBuffers (_film->audio_channels(), samples));
+				silence->make_silent ();
+				Audio (silence, t);
+			}
+			t += block;
+		}
+	}
+}
+
+DCPTime
+Player::one_video_frame () const
+{
+	return DCPTime::from_frames (1, _film->video_frame_rate ());
+}
diff --git a/src/lib/player.h b/src/lib/player.h
index 69149d039..c10f7adaa 100644
--- a/src/lib/player.h
+++ b/src/lib/player.h
@@ -107,6 +107,9 @@ private:
 	void image_subtitle (boost::weak_ptr<Piece>, ContentImageSubtitle);
 	void text_subtitle (boost::weak_ptr<Piece>, ContentTextSubtitle);
 	boost::shared_ptr<Resampler> resampler (boost::shared_ptr<const AudioContent> content, AudioStreamPtr stream, bool create);
+	DCPTime one_video_frame () const;
+	void fill_video (DCPTimePeriod period);
+	void fill_audio (DCPTimePeriod period);
 
 	boost::shared_ptr<const Film> _film;
 	boost::shared_ptr<const Playlist> _playlist;
@@ -154,6 +157,9 @@ private:
 	};
 	std::map<AudioStreamPtr, StreamState> _stream_states;
 
+	std::list<DCPTimePeriod> _no_video;
+	std::list<DCPTimePeriod> _no_audio;
+
 	std::list<std::pair<PlayerSubtitles, DCPTimePeriod> > _subtitles;
 
 	boost::shared_ptr<AudioProcessor> _audio_processor;
diff --git a/src/lib/upmixer_a.cc b/src/lib/upmixer_a.cc
index a1221e5ac..ca42cd386 100644
--- a/src/lib/upmixer_a.cc
+++ b/src/lib/upmixer_a.cc
@@ -73,7 +73,7 @@ UpmixerA::run (shared_ptr<const AudioBuffers> in, int channels)
 
 	/* Mix of L and R; -6dB down in amplitude (3dB in terms of power) */
 	shared_ptr<AudioBuffers> in_LR = in_L->clone ();
-	in_LR->accumulate_frames (in_R.get(), 0, 0, in_R->frames ());
+	in_LR->accumulate_frames (in_R.get(), in_R->frames(), 0, 0);
 	in_LR->apply_gain (-6);
 
 	/* Run filters */
diff --git a/src/lib/upmixer_b.cc b/src/lib/upmixer_b.cc
index 90e1267cd..2847da03b 100644
--- a/src/lib/upmixer_b.cc
+++ b/src/lib/upmixer_b.cc
@@ -68,7 +68,7 @@ UpmixerB::run (shared_ptr<const AudioBuffers> in, int channels)
 
 	/* L + R minus 6dB (in terms of amplitude) */
 	shared_ptr<AudioBuffers> in_LR = in->channel(0);
-	in_LR->accumulate_frames (in->channel(1).get(), 0, 0, in->frames());
+	in_LR->accumulate_frames (in->channel(1).get(), in->frames(), 0, 0);
 	in_LR->apply_gain (-6);
 
 	if (channels > 0) {