First cut.

8 files changed, 182 insertions, 175 deletions

diff --git a/src/lib/ffmpeg_decoder.cc b/src/lib/ffmpeg_decoder.cc
index 32c8e224a..f801821e9 100644
--- a/src/lib/ffmpeg_decoder.cc
+++ b/src/lib/ffmpeg_decoder.cc
@@ -80,10 +80,6 @@ FFmpegDecoder::FFmpegDecoder (shared_ptr<Film> f, DecodeOptions o)
 	setup_video ();
 	setup_audio ();
 	setup_subtitle ();
-
-	if (!o.video_sync) {
-		_first_video = 0;
-	}
 }
 
 FFmpegDecoder::~FFmpegDecoder ()
@@ -240,7 +236,7 @@ FFmpegDecoder::pass ()
 
 		if (_opt.decode_video) {
 			while (avcodec_decode_video2 (_video_codec_context, _frame, &frame_finished, &_packet) >= 0 && frame_finished) {
-				filter_and_emit_video (_frame);
+				filter_and_emit_video ();
 			}
 		}
 
@@ -265,16 +261,12 @@ FFmpegDecoder::pass ()
 				_film->log()->log (String::compose (N_("Used only %1 bytes of %2 in packet"), r, _packet.size));
 			}
 
-			if (_opt.video_sync) {
-				out_with_sync ();
-			} else {
-				filter_and_emit_video (_frame);
-			}
+			filter_and_emit_video ();
 		}
 
 	} else if (ffa && _packet.stream_index == ffa->id() && _opt.decode_audio) {
 		decode_audio_packet ();
-	} else if (_subtitle_stream && _packet.stream_index == _subtitle_stream->id() && _opt.decode_subtitles && _first_video) {
+	} else if (_subtitle_stream && _packet.stream_index == _subtitle_stream->id() && _opt.decode_subtitles) {
 
 		int got_subtitle;
 		AVSubtitle sub;
@@ -504,29 +496,29 @@ FFmpegDecoder::set_subtitle_stream (shared_ptr<SubtitleStream> s)
 }
 
 void
-FFmpegDecoder::filter_and_emit_video (AVFrame* frame)
+FFmpegDecoder::filter_and_emit_video ()
 {
 	boost::mutex::scoped_lock lm (_filter_graphs_mutex);
 	
 	shared_ptr<FilterGraph> graph;
 
 	list<shared_ptr<FilterGraph> >::iterator i = _filter_graphs.begin();
-	while (i != _filter_graphs.end() && !(*i)->can_process (libdcp::Size (frame->width, frame->height), (AVPixelFormat) frame->format)) {
+	while (i != _filter_graphs.end() && !(*i)->can_process (libdcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format)) {
 		++i;
 	}
 
 	if (i == _filter_graphs.end ()) {
-		graph.reset (new FilterGraph (_film, this, libdcp::Size (frame->width, frame->height), (AVPixelFormat) frame->format));
+		graph.reset (new FilterGraph (_film, this, libdcp::Size (_frame->width, _frame->height), (AVPixelFormat) _frame->format));
 		_filter_graphs.push_back (graph);
-		_film->log()->log (String::compose (N_("New graph for %1x%2, pixel format %3"), frame->width, frame->height, frame->format));
+		_film->log()->log (String::compose (N_("New graph for %1x%2, pixel format %3"), _frame->width, _frame->height, _frame->format));
 	} else {
 		graph = *i;
 	}
 
-	list<shared_ptr<Image> > images = graph->process (frame);
+	list<shared_ptr<Image> > images = graph->process (_frame);
 
 	for (list<shared_ptr<Image> >::iterator i = images.begin(); i != images.end(); ++i) {
-		emit_video (*i, frame_time ());
+		emit_video (*i, av_frame_get_best_effort_timestamp (_frame) * av_q2d (_format_context->streams[_video_stream]->time_base));
 	}
 }
 
@@ -614,53 +606,6 @@ FFmpegAudioStream::to_string () const
 }
 
 void
-FFmpegDecoder::out_with_sync ()
-{
-	/* Where we are in the output, in seconds */
-	double const out_pts_seconds = video_frame() / frames_per_second();
-	
-	/* Where we are in the source, in seconds */
-	double const source_pts_seconds = av_q2d (_format_context->streams[_packet.stream_index]->time_base)
-		* av_frame_get_best_effort_timestamp(_frame);
-	
-	_film->log()->log (
-		String::compose (N_("Source video frame ready; source at %1, output at %2"), source_pts_seconds, out_pts_seconds),
-		Log::VERBOSE
-		);
-	
-	if (!_first_video) {
-		_first_video = source_pts_seconds;
-	}
-	
-	/* Difference between where we are and where we should be */
-	double const delta = source_pts_seconds - _first_video.get() - out_pts_seconds;
-	double const one_frame = 1 / frames_per_second();
-	
-	/* Insert frames if required to get out_pts_seconds up to pts_seconds */
-	if (delta > one_frame) {
-		int const extra = rint (delta / one_frame);
-		for (int i = 0; i < extra; ++i) {
-			/* XXX: timestamp is wrong */
-			repeat_last_video (source_pts_seconds);
-			_film->log()->log (
-				String::compose (
-					N_("Extra video frame inserted at %1s; source frame %2, source PTS %3 (at %4 fps)"),
-					out_pts_seconds, video_frame(), source_pts_seconds, frames_per_second()
-					)
-				);
-		}
-	}
-	
-	if (delta > -one_frame) {
-		/* Process this frame */
-		filter_and_emit_video (_frame);
-	} else {
-		/* Otherwise we are omitting a frame to keep things right */
-		_film->log()->log (String::compose (N_("Frame removed at %1s"), out_pts_seconds));
-	}
-}
-
-void
 FFmpegDecoder::film_changed (Film::Property p)
 {
 	switch (p) {
@@ -685,12 +630,6 @@ FFmpegDecoder::length () const
 	return (double(_format_context->duration) / AV_TIME_BASE) * frames_per_second();
 }
 
-double
-FFmpegDecoder::frame_time () const
-{
-	return av_frame_get_best_effort_timestamp(_frame) * av_q2d (_format_context->streams[_video_stream]->time_base);
-}
-
 void
 FFmpegDecoder::decode_audio_packet ()
 {
@@ -707,54 +646,21 @@ FFmpegDecoder::decode_audio_packet ()
 
 		int frame_finished;
 		int const decode_result = avcodec_decode_audio4 (_audio_codec_context, _frame, &frame_finished, &copy_packet);
-		if (decode_result >= 0 && frame_finished) {
-			
-			/* Where we are in the source, in seconds */
-			double const source_pts_seconds = av_q2d (_format_context->streams[copy_packet.stream_index]->time_base)
-				* av_frame_get_best_effort_timestamp(_frame);
-			
-			/* We only decode audio if we've had our first video packet through, and if it
-			   was before this packet.  Until then audio is thrown away.
-			*/
+		if (decode_result >= 0) {
+			if (frame_finished) {
 			
-			if ((_first_video && _first_video.get() <= source_pts_seconds) || !_opt.decode_video) {
-				
-				if (!_first_audio && _opt.decode_video) {
-					_first_audio = source_pts_seconds;
-					
-					/* This is our first audio frame, and if we've arrived here we must have had our
-					   first video frame.  Push some silence to make up any gap between our first
-					   video frame and our first audio.
-					*/
-					
-					/* frames of silence that we must push */
-					int const s = rint ((_first_audio.get() - _first_video.get()) * ffa->sample_rate ());
-					
-					_film->log()->log (
-						String::compose (
-							N_("First video at %1, first audio at %2, pushing %3 audio frames of silence for %4 channels (%5 bytes per sample)"),
-							_first_video.get(), _first_audio.get(), s, ffa->channels(), bytes_per_audio_sample()
-							)
-						);
-					
-					if (s) {
-						shared_ptr<AudioBuffers> audio (new AudioBuffers (ffa->channels(), s));
-						audio->make_silent ();
-						/* XXX: this time stamp is wrong */
-						Audio (audio, source_pts_seconds);
-					}
-				}
+				/* Where we are in the source, in seconds */
+				double const source_pts_seconds = av_q2d (_format_context->streams[copy_packet.stream_index]->time_base)
+					* av_frame_get_best_effort_timestamp(_frame);
 				
 				int const data_size = av_samples_get_buffer_size (
 					0, _audio_codec_context->channels, _frame->nb_samples, audio_sample_format (), 1
 					);
 				
 				assert (_audio_codec_context->channels == _film->audio_channels());
-				Audio (deinterleave_audio (_frame->data, data_size), source_pts_seconds );
+				Audio (deinterleave_audio (_frame->data, data_size), source_pts_seconds);
 			}
-		}
-
-		if (decode_result >= 0) {
+			
 			copy_packet.data += decode_result;
 			copy_packet.size -= decode_result;
 		}
diff --git a/src/lib/ffmpeg_decoder.h b/src/lib/ffmpeg_decoder.h
index 1bb14ce9c..2a4d40b1d 100644
--- a/src/lib/ffmpeg_decoder.h
+++ b/src/lib/ffmpeg_decoder.h
@@ -111,9 +111,7 @@ private:
 	AVSampleFormat audio_sample_format () const;
 	int bytes_per_audio_sample () const;
 
-	void out_with_sync ();
-	void filter_and_emit_video (AVFrame *);
-	double frame_time () const;
+	void filter_and_emit_video ();
 
 	void setup_general ();
 	void setup_video ();
@@ -143,9 +141,6 @@ private:
 
 	AVPacket _packet;
 
-	boost::optional<double> _first_video;
-	boost::optional<double> _first_audio;
-
 	std::list<boost::shared_ptr<FilterGraph> > _filter_graphs;
 	boost::mutex _filter_graphs_mutex;
 };
diff --git a/src/lib/imagemagick_decoder.cc b/src/lib/imagemagick_decoder.cc
index 38dace6de..119f05792 100644
--- a/src/lib/imagemagick_decoder.cc
+++ b/src/lib/imagemagick_decoder.cc
@@ -31,6 +31,8 @@ using std::cout;
 using boost::shared_ptr;
 using libdcp::Size;
 
+/* XXX: reads a directory and then ignores it */
+
 ImageMagickDecoder::ImageMagickDecoder (
 	boost::shared_ptr<Film> f, DecodeOptions o)
 	: Decoder (f, o)
@@ -77,8 +79,8 @@ ImageMagickDecoder::pass ()
 			return true;
 		}
 
-		/* XXX: timestamp is wrong */
-		repeat_last_video (0);
+		/* XXX: timestamp */
+		emit_video (_image, 0);
 		return false;
 	}
 	
@@ -101,9 +103,10 @@ ImageMagickDecoder::pass ()
 
 	delete magick_image;
 
-	image = image->crop (_film->crop(), true);
-	
-	emit_video (image, 0);
+	_image = image->crop (_film->crop(), true);
+
+	/* XXX: timestamp */
+	emit_video (_image, 0);
 
 	++_iter;
 	return false;
@@ -131,6 +134,7 @@ ImageMagickDecoder::seek_to_last ()
 bool
 ImageMagickDecoder::seek (double t)
 {
+	/* XXX: frames_per_second == 0 */
 	int const f = t * frames_per_second();
 	
 	_iter = _files.begin ();
diff --git a/src/lib/imagemagick_decoder.h b/src/lib/imagemagick_decoder.h
index 0e375f6e9..ef550c651 100644
--- a/src/lib/imagemagick_decoder.h
+++ b/src/lib/imagemagick_decoder.h
@@ -86,4 +86,7 @@ private:
 	
 	std::list<std::string> _files;
 	std::list<std::string>::iterator _iter;
+
+	boost::shared_ptr<Image> _image;
+	
 };
diff --git a/src/lib/matcher.cc b/src/lib/matcher.cc
index 3a513b24e..70a9b2a85 100644
--- a/src/lib/matcher.cc
+++ b/src/lib/matcher.cc
@@ -24,6 +24,8 @@
 #include "i18n.h"
 
 using std::min;
+using std::cout;
+using std::list;
 using boost::shared_ptr;
 
 Matcher::Matcher (Log* log, int sample_rate, float frames_per_second)
@@ -37,22 +39,72 @@ Matcher::Matcher (Log* log, int sample_rate, float frames_per_second)
 }
 
 void
-Matcher::process_video (boost::shared_ptr<Image> i, bool same, boost::shared_ptr<Subtitle> s, double)
+Matcher::process_video (boost::shared_ptr<Image> image, bool same, boost::shared_ptr<Subtitle> sub, double t)
 {
-	Video (i, same, s);
-	_video_frames++;
+	_pixel_format = image->pixel_format ();
+	_size = image->size ();
 
-	_pixel_format = i->pixel_format ();
-	_size = i->size ();
+	if (!_first_input) {
+		_first_input = t;
+	}
+	
+	if (_audio_frames == 0 && _pending_audio.empty ()) {
+		/* No audio yet; we must postpone this frame until we have some */
+		_pending_video.push_back (VideoRecord (image, same, sub, t));
+	} else if (!_pending_audio.empty() && _pending_video.empty ()) {
+		/* First video since we got audio */
+		_pending_video.push_back (VideoRecord (image, same, sub, t));
+		fix_start ();
+	} else {
+		/* Normal running */
+
+		/* Difference between where this video is and where it should be */
+		double const delta = t - _first_input.get() - _video_frames / _frames_per_second;
+		double const one_frame = 1 / _frames_per_second;
+
+		if (delta > one_frame) {
+			/* Insert frames to make up the difference */
+			int const extra = rint (delta / one_frame);
+			for (int i = 0; i < extra; ++i) {
+				repeat_last_video ();
+				_log->log (String::compose ("Extra video frame inserted at %1s", _video_frames / _frames_per_second));
+			}
+		}
+
+		if (delta > -one_frame) {
+			Video (image, same, sub);
+			++_video_frames;
+		} else {
+			/* We are omitting a frame to keep things right */
+			_log->log (String::compose ("Frame removed at %1s", t));
+		}
+	}
+		
+	_last_image = image;
+	_last_subtitle = sub;
 }
 
 void
-Matcher::process_audio (boost::shared_ptr<AudioBuffers> b, double)
+Matcher::process_audio (boost::shared_ptr<AudioBuffers> b, double t)
 {
-	Audio (b);
-	_audio_frames += b->frames ();
-
 	_channels = b->channels ();
+	
+	if (!_first_input) {
+		_first_input = t;
+	}
+	
+	if (_video_frames == 0 && _pending_video.empty ()) {
+		/* No video yet; we must postpone these data until we have some */
+		_pending_audio.push_back (AudioRecord (b, t));
+	} else if (!_pending_video.empty() && _pending_audio.empty ()) {
+		/* First audio since we got video */
+		_pending_audio.push_back (AudioRecord (b, t));
+		fix_start ();
+	} else {
+		/* Normal running.  We assume audio time stamps are consecutive */
+		Audio (b);
+		_audio_frames += b->frames ();
+	}
 }
 
 void
@@ -63,38 +115,58 @@ Matcher::process_end ()
 		return;
 	}
 	
-	int64_t audio_short_by_frames = video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second) - _audio_frames;
-
-	_log->log (
-		String::compose (
-			N_("Matching processor has seen %1 video frames (which equals %2 audio frames) and %3 audio frames"),
-			_video_frames,
-			video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second),
-			_audio_frames
-			)
-		);
+	match ((double (_audio_frames) / _sample_rate) - (double (_video_frames) / _frames_per_second));
+}
+
+void
+Matcher::fix_start ()
+{
+	assert (!_pending_video.empty ());
+	assert (!_pending_audio.empty ());
+
+	_log->log (String::compose ("Fixing start; video at %1, audio at %2", _pending_video.front().time, _pending_audio.front().time));
+
+	match (_pending_video.front().time - _pending_audio.front().time);
+
+	for (list<VideoRecord>::iterator i = _pending_video.begin(); i != _pending_video.end(); ++i) {
+		Video (i->image, i->same, i->subtitle);
+	}
+
+	for (list<AudioRecord>::iterator i = _pending_audio.begin(); i != _pending_audio.end(); ++i) {
+		Audio (i->audio);
+	}
 	
-	if (audio_short_by_frames < 0) {
-		
-		_log->log (String::compose (N_("%1 too many audio frames"), -audio_short_by_frames));
-		
-		/* We have seen more audio than video.  Emit enough black video frames so that we reverse this */
-		int const black_video_frames = ceil (-audio_short_by_frames * _frames_per_second / _sample_rate);
+	_pending_video.clear ();
+	_pending_audio.clear ();
+}
+
+void
+Matcher::match (double extra_video_needed)
+{
+	if (extra_video_needed) {
+
+		/* Emit black video frames */
 		
+		int const black_video_frames = ceil (extra_video_needed * _frames_per_second);
+
 		_log->log (String::compose (N_("Emitting %1 frames of black video"), black_video_frames));
 
 		shared_ptr<Image> black (new SimpleImage (_pixel_format.get(), _size.get(), true));
 		black->make_black ();
 		for (int i = 0; i < black_video_frames; ++i) {
 			Video (black, i != 0, shared_ptr<Subtitle>());
+			++_video_frames;
 		}
-		
-		/* Now recompute our check value */
-		audio_short_by_frames = video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second) - _audio_frames;
+
+		extra_video_needed -= black_video_frames / _frames_per_second;
 	}
-	
-	if (audio_short_by_frames > 0) {
-		_log->log (String::compose (N_("Emitted %1 too few audio frames"), audio_short_by_frames));
+
+	if (extra_video_needed < 0) {
+		
+		/* Emit silence */
+		
+		int64_t to_do = rint (-extra_video_needed * _sample_rate);
+		_log->log (String::compose (N_("Emitted %1 frames of silence"), to_do));
 
 		/* Do things in half second blocks as I think there may be limits
 		   to what FFmpeg (and in particular the resampler) can cope with.
@@ -103,7 +175,6 @@ Matcher::process_end ()
 		shared_ptr<AudioBuffers> b (new AudioBuffers (_channels.get(), block));
 		b->make_silent ();
 		
-		int64_t to_do = audio_short_by_frames;
 		while (to_do > 0) {
 			int64_t const this_time = min (to_do, block);
 			b->set_frames (this_time);
@@ -113,3 +184,16 @@ Matcher::process_end ()
 		}
 	}
 }
+
+void
+Matcher::repeat_last_video ()
+{
+	if (!_last_image) {
+		_last_image.reset (new SimpleImage (_pixel_format.get(), _size.get(), true));
+		_last_image->make_black ();
+	}
+
+	Video (_last_image, true, _last_subtitle);
+	++_video_frames;
+}
+
diff --git a/src/lib/matcher.h b/src/lib/matcher.h
index 4a66f4e70..2f580b589 100644
--- a/src/lib/matcher.h
+++ b/src/lib/matcher.h
@@ -30,6 +30,10 @@ public:
 	void process_end ();
 
 private:
+	void fix_start ();
+	void match (double);
+	void repeat_last_video ();
+	
 	int _sample_rate;
 	float _frames_per_second;
 	int _video_frames;
@@ -37,4 +41,36 @@ private:
 	boost::optional<AVPixelFormat> _pixel_format;
 	boost::optional<libdcp::Size> _size;
 	boost::optional<int> _channels;
+
+	struct VideoRecord {
+		VideoRecord (boost::shared_ptr<Image> i, bool s, boost::shared_ptr<Subtitle> sub, double t)
+			: image (i)
+			, same (s)
+			, subtitle (sub)
+			, time (t)
+		{}
+		
+		boost::shared_ptr<Image> image;
+		bool same;
+		boost::shared_ptr<Subtitle> subtitle;
+		double time;
+	};
+
+	std::list<VideoRecord> _pending_video;
+
+	struct AudioRecord {
+		AudioRecord (boost::shared_ptr<AudioBuffers> a, double t)
+			: audio (a)
+			, time (t)
+		{}
+		
+		boost::shared_ptr<AudioBuffers> audio;
+		double time;
+	};
+
+	std::list<AudioRecord> _pending_audio;
+
+	boost::optional<double> _first_input;
+	boost::shared_ptr<Image> _last_image;
+	boost::shared_ptr<Subtitle> _last_subtitle;
 };
diff --git a/src/lib/video_decoder.cc b/src/lib/video_decoder.cc
index 773688b34..7fff93c45 100644
--- a/src/lib/video_decoder.cc
+++ b/src/lib/video_decoder.cc
@@ -27,6 +27,7 @@
 
 #include "i18n.h"
 
+using std::cout;
 using boost::shared_ptr;
 using boost::optional;
 
@@ -55,21 +56,6 @@ VideoDecoder::emit_video (shared_ptr<Image> image, double t)
 	_last_source_time = t;
 }
 
-/** Called by subclasses to repeat the last video frame that we
- *  passed to emit_video().  If emit_video hasn't yet been called,
- *  we will generate a black frame.
- */
-void
-VideoDecoder::repeat_last_video (double t)
-{
-	if (!_last_image) {
-		_last_image.reset (new SimpleImage (pixel_format(), native_size(), true));
-		_last_image->make_black ();
-	}
-
-	signal_video (_last_image, true, _last_subtitle, t);
-}
-
 /** Emit our signal to say that some video data is ready.
  *  @param image Video frame.
  *  @param same true if `image' is the same as the last one we emitted.
@@ -81,9 +67,6 @@ VideoDecoder::signal_video (shared_ptr<Image> image, bool same, shared_ptr<Subti
 	TIMING (N_("Decoder emits %1"), _video_frame);
 	Video (image, same, sub, t);
 	++_video_frame;
-
-	_last_image = image;
-	_last_subtitle = sub;
 }
 
 /** Set up the current subtitle.  This will be put onto frames that
diff --git a/src/lib/video_decoder.h b/src/lib/video_decoder.h
index 5e9c60d08..7d43c9e82 100644
--- a/src/lib/video_decoder.h
+++ b/src/lib/video_decoder.h
@@ -67,7 +67,6 @@ protected:
 
 	void emit_video (boost::shared_ptr<Image>, double);
 	void emit_subtitle (boost::shared_ptr<TimedSubtitle>);
-	void repeat_last_video (double t);
 
 	/** Subtitle stream to use when decoding */
 	boost::shared_ptr<SubtitleStream> _subtitle_stream;
@@ -81,9 +80,6 @@ private:
 	double _last_source_time;
 	
 	boost::shared_ptr<TimedSubtitle> _timed_subtitle;
-
-	boost::shared_ptr<Image> _last_image;
-	boost::shared_ptr<Subtitle> _last_subtitle;
 };
 
 #endif