Don't allow the queue to get too big with REPEAT frames otherwisev2.13.26

there's a long delay at the end of the job while they are written.
We must still write FULL frames even if the queue is long (we only
stop doing that if the queue has too many FULL frames i.e. too much
memory consumption).  With this commit we stop writing REPEAT/FAKE
frames when the queue gets long and assume there will always be a
sequence image for writing and hence the main writer thread will reduce
the queue given time.

3 files changed, 14 insertions, 6 deletions

diff --git a/ChangeLog b/ChangeLog
index 386173b96..f740d2e70 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2018-06-05  Carl Hetherington  <cth@carlh.net>
+
+	* Don't allow the queue to get too big with repeat frames
+	otherwise there's a long delay at the end of the job
+	while they are written (#1317).
+
 2018-06-04  Carl Hetherington  <cth@carlh.net>
 
 	* Updated nl_NL translation from Rob van Nieuwkerk.
diff --git a/src/lib/writer.cc b/src/lib/writer.cc
index 896d59cfc..39d6f6118 100644
--- a/src/lib/writer.cc
+++ b/src/lib/writer.cc
@@ -75,6 +75,7 @@ Writer::Writer (shared_ptr<const Film> film, weak_ptr<Job> j)
 	, _finish (false)
 	, _queued_full_in_memory (0)
 	, _maximum_frames_in_memory (0)
+	, _maximum_queue_size (0)
 	, _full_written (0)
 	, _fake_written (0)
 	, _repeat_written (0)
@@ -128,7 +129,7 @@ Writer::write (Data encoded, Frame frame, Eyes eyes)
 	boost::mutex::scoped_lock lock (_state_mutex);
 
 	while (_queued_full_in_memory > _maximum_frames_in_memory) {
-		/* The queue is too big; wait until that is sorted out */
+		/* There are too many full frames in memory; wait until that is sorted out */
 		_full_condition.wait (lock);
 	}
 
@@ -171,7 +172,7 @@ Writer::repeat (Frame frame, Eyes eyes)
 {
 	boost::mutex::scoped_lock lock (_state_mutex);
 
-	while (_queued_full_in_memory > _maximum_frames_in_memory) {
+	while (_queue.size() > _maximum_queue_size) {
 		/* The queue is too big; wait until that is sorted out */
 		_full_condition.wait (lock);
 	}
@@ -199,7 +200,7 @@ Writer::fake_write (Frame frame, Eyes eyes)
 {
 	boost::mutex::scoped_lock lock (_state_mutex);
 
-	while (_queued_full_in_memory > _maximum_frames_in_memory) {
+	while (_queue.size() > _maximum_queue_size) {
 		/* The queue is too big; wait until that is sorted out */
 		_full_condition.wait (lock);
 	}
@@ -405,6 +406,7 @@ try
 			}
 
 			lock.lock ();
+			_full_condition.notify_all ();
 		}
 
 		while (_queued_full_in_memory > _maximum_frames_in_memory) {
@@ -440,10 +442,8 @@ try
 			lock.lock ();
 			i->encoded.reset ();
 			--_queued_full_in_memory;
+			_full_condition.notify_all ();
 		}
-
-		/* The queue has probably just gone down a bit; notify anything wait()ing on _full_condition */
-		_full_condition.notify_all ();
 	}
 }
 catch (...)
@@ -713,6 +713,7 @@ void
 Writer::set_encoder_threads (int threads)
 {
 	_maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
+	_maximum_queue_size = lrint (threads * 16);
 }
 
 void
diff --git a/src/lib/writer.h b/src/lib/writer.h
index ec7b98804..0e1c0e02f 100644
--- a/src/lib/writer.h
+++ b/src/lib/writer.h
@@ -144,6 +144,7 @@ private:
 	 *  ordering
 	 */
 	int _maximum_frames_in_memory;
+	unsigned int _maximum_queue_size;
 
 	/** number of FULL written frames */
 	int _full_written;