X-Git-Url: https://git.carlh.net/gitweb/?a=blobdiff_plain;f=src%2Flib%2Fmatcher.cc;h=c56a563015b6eb617bf38535ff81be81c7cffeb0;hb=f861018389acd9d277fe34d7621182b9b54f977f;hp=182fb306c695d1008b4b8be8d50732af4cddda55;hpb=420adb1fd2910fd24eb84be98169afc209f76a0e;p=dcpomatic.git diff --git a/src/lib/matcher.cc b/src/lib/matcher.cc index 182fb306c..c56a56301 100644 --- a/src/lib/matcher.cc +++ b/src/lib/matcher.cc @@ -21,36 +21,104 @@ #include "image.h" #include "log.h" +#include "i18n.h" + using std::min; +using std::cout; +using std::list; using boost::shared_ptr; -Matcher::Matcher (Log* log, int sample_rate, float frames_per_second) - : AudioVideoProcessor (log) +Matcher::Matcher (shared_ptr log, int sample_rate, float frames_per_second) + : Processor (log) , _sample_rate (sample_rate) , _frames_per_second (frames_per_second) , _video_frames (0) , _audio_frames (0) + , _had_first_video (false) + , _had_first_audio (false) { } void -Matcher::process_video (boost::shared_ptr i, bool same, boost::shared_ptr s) +Matcher::process_video (shared_ptr image, bool same, boost::shared_ptr sub, double t) { - Video (i, same, s); - _video_frames++; + _pixel_format = image->pixel_format (); + _size = image->size (); + + _log->log(String::compose("Matcher video @ %1 [audio=%2, video=%3, pending_audio=%4]", t, _audio_frames, _video_frames, _pending_audio.size())); + + if (!_first_input) { + _first_input = t; + } + + bool const this_is_first_video = !_had_first_video; + _had_first_video = true; + + if (this_is_first_video && _had_first_audio) { + /* First video since we got audio */ + fix_start (t); + } - _pixel_format = i->pixel_format (); - _size = i->size (); + /* Video before audio is fine, since we can make up an arbitrary difference + with audio samples (contrasting with video which is quantised to frames) + */ + + /* Difference between where this video is and where it should be */ + double const delta = t - _first_input.get() - _video_frames / _frames_per_second; + double const one_frame = 1 / _frames_per_second; + + if (delta > one_frame) { + /* Insert frames to make up the difference */ + int const extra = rint (delta / one_frame); + for (int i = 0; i < extra; ++i) { + repeat_last_video (); + _log->log (String::compose ("Extra video frame inserted at %1s", _video_frames / _frames_per_second)); + } + } + + if (delta > -one_frame) { + Video (image, same, sub); + ++_video_frames; + } else { + /* We are omitting a frame to keep things right */ + _log->log (String::compose ("Frame removed at %1s", t)); + } + + _last_image = image; + _last_subtitle = sub; } void -Matcher::process_audio (boost::shared_ptr b) +Matcher::process_audio (shared_ptr b, double t) { - Audio (b); - _audio_frames += b->frames (); - _channels = b->channels (); + + _log->log (String::compose ( + "Matcher audio (%1 frames) @ %2 [video=%3, audio=%4, pending_audio=%5]", + b->frames(), t, _video_frames, _audio_frames, _pending_audio.size() + ) + ); + + if (!_first_input) { + _first_input = t; + } + + bool const this_is_first_audio = _had_first_audio; + _had_first_audio = true; + + if (!_had_first_video) { + /* No video yet; we must postpone these data until we have some */ + _pending_audio.push_back (AudioRecord (b, t)); + } else if (this_is_first_audio && !_had_first_video) { + /* First audio since we got video */ + _pending_audio.push_back (AudioRecord (b, t)); + fix_start (_first_input.get ()); + } else { + /* Normal running. We assume audio time stamps are consecutive */ + Audio (b); + _audio_frames += b->frames (); + } } void @@ -60,39 +128,58 @@ Matcher::process_end () /* We won't do anything */ return; } + + _log->log (String::compose ("Matcher has seen %1 video frames (which equals %2 audio frames) and %3 audio frames", + _video_frames, video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second), _audio_frames)); - int64_t audio_short_by_frames = video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second) - _audio_frames; - - _log->log ( - String::compose ( - "Matching processor has seen %1 video frames (which equals %2 audio frames) and %3 audio frames", - _video_frames, - video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second), - _audio_frames - ) - ); + match ((double (_audio_frames) / _sample_rate) - (double (_video_frames) / _frames_per_second)); +} + +void +Matcher::fix_start (double first_video) +{ + assert (!_pending_audio.empty ()); + + _log->log (String::compose ("Fixing start; video at %1, audio at %2", first_video, _pending_audio.front().time)); + + match (first_video - _pending_audio.front().time); + + for (list::iterator i = _pending_audio.begin(); i != _pending_audio.end(); ++i) { + process_audio (i->audio, i->time); + } - if (audio_short_by_frames < 0) { - - _log->log (String::compose ("%1 too many audio frames", -audio_short_by_frames)); - - /* We have seen more audio than video. Emit enough black video frames so that we reverse this */ - int const black_video_frames = ceil (-audio_short_by_frames * _frames_per_second / _sample_rate); + _pending_audio.clear (); +} + +void +Matcher::match (double extra_video_needed) +{ + _log->log (String::compose ("Match %1", extra_video_needed)); + + if (extra_video_needed > 0) { + + /* Emit black video frames */ - _log->log (String::compose ("Emitting %1 frames of black video", black_video_frames)); + int const black_video_frames = ceil (extra_video_needed * _frames_per_second); + + _log->log (String::compose (N_("Emitting %1 frames of black video"), black_video_frames)); shared_ptr black (new SimpleImage (_pixel_format.get(), _size.get(), true)); black->make_black (); for (int i = 0; i < black_video_frames; ++i) { Video (black, i != 0, shared_ptr()); + ++_video_frames; } - - /* Now recompute our check value */ - audio_short_by_frames = video_frames_to_audio_frames (_video_frames, _sample_rate, _frames_per_second) - _audio_frames; + + extra_video_needed -= black_video_frames / _frames_per_second; } - - if (audio_short_by_frames > 0) { - _log->log (String::compose ("Emitted %1 too few audio frames", audio_short_by_frames)); + + if (extra_video_needed < 0) { + + /* Emit silence */ + + int64_t to_do = -extra_video_needed * _sample_rate; + _log->log (String::compose (N_("Emitting %1 frames of silence"), to_do)); /* Do things in half second blocks as I think there may be limits to what FFmpeg (and in particular the resampler) can cope with. @@ -101,7 +188,6 @@ Matcher::process_end () shared_ptr b (new AudioBuffers (_channels.get(), block)); b->make_silent (); - int64_t to_do = audio_short_by_frames; while (to_do > 0) { int64_t const this_time = min (to_do, block); b->set_frames (this_time); @@ -111,3 +197,17 @@ Matcher::process_end () } } } + +void +Matcher::repeat_last_video () +{ + if (!_last_image) { + shared_ptr im (new SimpleImage (_pixel_format.get(), _size.get(), true)); + im->make_black (); + _last_image = im; + } + + Video (_last_image, true, _last_subtitle); + ++_video_frames; +} +