Fix some spelling mistakes (mostly in comments).
[dcpomatic.git] / src / lib / writer.cc
index f126d5b62c010958ecc328996a4d14eec2c99774..2dd46f0b2324f4516364b319ab2a1561542cba26 100644 (file)
 /*
-    Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
 
-    This program is free software; you can redistribute it and/or modify
+    This file is part of DCP-o-matic.
+
+    DCP-o-matic is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
 
-    This program is distributed in the hope that it will be useful,
+    DCP-o-matic is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+    along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
 
 */
 
-#include "writer.h"
-#include "compose.hpp"
-#include "film.h"
-#include "ratio.h"
-#include "log.h"
-#include "dcp_video.h"
-#include "dcp_content_type.h"
+
+#include "audio_buffers.h"
 #include "audio_mapping.h"
+#include "compose.hpp"
 #include "config.h"
-#include "job.h"
 #include "cross.h"
-#include "audio_buffers.h"
-#include "md5_digester.h"
-#include "encoded_data.h"
+#include "dcp_content_type.h"
+#include "dcp_video.h"
+#include "dcpomatic_log.h"
+#include "film.h"
+#include "job.h"
+#include "log.h"
+#include "ratio.h"
+#include "reel_writer.h"
+#include "text_content.h"
+#include "util.h"
 #include "version.h"
-#include "font.h"
-#include <dcp/mono_picture_mxf.h>
-#include <dcp/stereo_picture_mxf.h>
-#include <dcp/sound_mxf.h>
-#include <dcp/sound_mxf_writer.h>
-#include <dcp/reel.h>
-#include <dcp/reel_mono_picture_asset.h>
-#include <dcp/reel_stereo_picture_asset.h>
-#include <dcp/reel_sound_asset.h>
-#include <dcp/reel_subtitle_asset.h>
-#include <dcp/dcp.h>
+#include "writer.h"
 #include <dcp/cpl.h>
-#include <dcp/signer.h>
-#include <dcp/interop_subtitle_content.h>
-#include <fstream>
+#include <dcp/locale_convert.h>
+#include <dcp/raw_convert.h>
+#include <dcp/reel_file_asset.h>
 #include <cerrno>
+#include <cfloat>
+#include <set>
 
 #include "i18n.h"
 
-#define LOG_GENERAL(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_GENERAL);
-#define LOG_TIMING(...) _film->log()->microsecond_log (String::compose (__VA_ARGS__), Log::TYPE_TIMING);
-#define LOG_WARNING_NC(...) _film->log()->log (__VA_ARGS__, Log::TYPE_WARNING);
-#define LOG_WARNING(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_WARNING);
-#define LOG_ERROR(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_ERROR);
-#define LOG_DEBUG(...) _film->log()->log (String::compose (__VA_ARGS__), Log::TYPE_DEBUG);
-#define LOG_DEBUG_NC(...) _film->log()->log (__VA_ARGS__, Log::TYPE_DEBUG);
 
 /* OS X strikes again */
 #undef set_key
 
-using std::make_pair;
-using std::pair;
-using std::string;
-using std::list;
-using std::cout;
-using boost::shared_ptr;
-using boost::weak_ptr;
-using boost::dynamic_pointer_cast;
-
-int const Writer::_maximum_frames_in_memory = Config::instance()->num_local_encoding_threads() + 4;
 
-Writer::Writer (shared_ptr<const Film> f, weak_ptr<Job> j)
-       : _film (f)
+using std::cout;
+using std::dynamic_pointer_cast;
+using std::make_shared;
+using std::max;
+using std::min;
+using std::shared_ptr;
+using std::set;
+using std::string;
+using std::vector;
+using std::weak_ptr;
+using boost::optional;
+#if BOOST_VERSION >= 106100
+using namespace boost::placeholders;
+#endif
+using dcp::Data;
+using dcp::ArrayData;
+using namespace dcpomatic;
+
+
+/** @param j Job to report progress to, or 0.
+ *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
+ */
+Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
+       : WeakConstFilm (weak_film)
        , _job (j)
-       , _first_nonexistant_frame (0)
-       , _thread (0)
-       , _finish (false)
-       , _queued_full_in_memory (0)
-       , _last_written_frame (-1)
-       , _last_written_eyes (EYES_RIGHT)
-       , _full_written (0)
-       , _fake_written (0)
-       , _pushed_to_disk (0)
+       /* These will be reset to sensible values when J2KEncoder is created */
+       , _maximum_frames_in_memory (8)
+       , _maximum_queue_size (8)
+       , _text_only (text_only)
 {
-       /* Remove any old DCP */
-       boost::filesystem::remove_all (_film->dir (_film->dcp_name ()));
+       auto job = _job.lock ();
 
-       shared_ptr<Job> job = _job.lock ();
-       DCPOMATIC_ASSERT (job);
+       int reel_index = 0;
+       auto const reels = film()->reels();
+       for (auto p: reels) {
+               _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
+       }
 
-       job->sub (_("Checking existing image data"));
-       check_existing_picture_mxf ();
+       _last_written.resize (reels.size());
 
-       /* Create our picture asset in a subdirectory, named according to those
-          film's parameters which affect the video output.  We will hard-link
-          it into the DCP later.
+       /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
+          and captions arrive to the Writer in sequence.  This is not so for video.
        */
-
-       if (_film->three_d ()) {
-               _picture_mxf.reset (new dcp::StereoPictureMXF (dcp::Fraction (_film->video_frame_rate (), 1)));
-       } else {
-               _picture_mxf.reset (new dcp::MonoPictureMXF (dcp::Fraction (_film->video_frame_rate (), 1)));
+       _audio_reel = _reels.begin ();
+       _subtitle_reel = _reels.begin ();
+       for (auto i: film()->closed_caption_tracks()) {
+               _caption_reels[i] = _reels.begin ();
        }
+       _atmos_reel = _reels.begin ();
 
-       _picture_mxf->set_size (_film->frame_size ());
-
-       if (_film->encrypted ()) {
-               _picture_mxf->set_key (_film->key ());
+       /* Check that the signer is OK */
+       string reason;
+       if (!Config::instance()->signer_chain()->valid(&reason)) {
+               throw InvalidSignerError (reason);
        }
-       
-       _picture_mxf_writer = _picture_mxf->start_write (
-               _film->internal_video_mxf_dir() / _film->internal_video_mxf_filename(),
-               _film->interop() ? dcp::INTEROP : dcp::SMPTE,
-               _first_nonexistant_frame > 0
-               );
-
-       if (_film->audio_channels ()) {
-               _sound_mxf.reset (new dcp::SoundMXF (dcp::Fraction (_film->video_frame_rate(), 1), _film->audio_frame_rate (), _film->audio_channels ()));
+}
 
-               if (_film->encrypted ()) {
-                       _sound_mxf->set_key (_film->key ());
-               }
-       
-               /* Write the sound MXF into the film directory so that we leave the creation
-                  of the DCP directory until the last minute.
-               */
-               _sound_mxf_writer = _sound_mxf->start_write (_film->directory() / _film->audio_mxf_filename(), _film->interop() ? dcp::INTEROP : dcp::SMPTE);
-       }
 
-       /* Check that the signer is OK if we need one */
-       if (_film->is_signed() && !Config::instance()->signer()->valid ()) {
-               throw InvalidSignerError ();
+void
+Writer::start ()
+{
+       if (!_text_only) {
+               _thread = boost::thread (boost::bind(&Writer::thread, this));
+#ifdef DCPOMATIC_LINUX
+               pthread_setname_np (_thread.native_handle(), "writer");
+#endif
        }
-
-       _thread = new boost::thread (boost::bind (&Writer::thread, this));
-
-       job->sub (_("Encoding image data"));
 }
 
+
 Writer::~Writer ()
 {
-       terminate_thread (false);
+       if (!_text_only) {
+               terminate_thread (false);
+       }
 }
 
+
+/** Pass a video frame to the writer for writing to disk at some point.
+ *  This method can be called with frames out of order.
+ *  @param encoded JPEG2000-encoded data.
+ *  @param frame Frame index within the DCP.
+ *  @param eyes Eyes that this frame image is for.
+ */
 void
-Writer::write (shared_ptr<const EncodedData> encoded, int frame, Eyes eyes)
+Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 {
-       boost::mutex::scoped_lock lock (_mutex);
+       boost::mutex::scoped_lock lock (_state_mutex);
 
        while (_queued_full_in_memory > _maximum_frames_in_memory) {
-               /* The queue is too big; wait until that is sorted out */
+               /* There are too many full frames in memory; wake the main writer thread and
+                  wait until it sorts everything out */
+               _empty_condition.notify_all ();
                _full_condition.wait (lock);
        }
 
        QueueItem qi;
-       qi.type = QueueItem::FULL;
+       qi.type = QueueItem::Type::FULL;
        qi.encoded = encoded;
-       qi.frame = frame;
+       qi.reel = video_reel (frame);
+       qi.frame = frame - _reels[qi.reel].start ();
 
-       if (_film->three_d() && eyes == EYES_BOTH) {
+       if (film()->three_d() && eyes == Eyes::BOTH) {
                /* 2D material in a 3D DCP; fake the 3D */
-               qi.eyes = EYES_LEFT;
+               qi.eyes = Eyes::LEFT;
                _queue.push_back (qi);
                ++_queued_full_in_memory;
-               qi.eyes = EYES_RIGHT;
+               qi.eyes = Eyes::RIGHT;
                _queue.push_back (qi);
                ++_queued_full_in_memory;
        } else {
@@ -179,28 +171,80 @@ Writer::write (shared_ptr<const EncodedData> encoded, int frame, Eyes eyes)
        _empty_condition.notify_all ();
 }
 
+
+bool
+Writer::can_repeat (Frame frame) const
+{
+       return frame > _reels[video_reel(frame)].start();
+}
+
+
+/** Repeat the last frame that was written to a reel as a new frame.
+ *  @param frame Frame index within the DCP of the new (repeated) frame.
+ *  @param eyes Eyes that this repeated frame image is for.
+ */
 void
-Writer::fake_write (int frame, Eyes eyes)
+Writer::repeat (Frame frame, Eyes eyes)
 {
-       boost::mutex::scoped_lock lock (_mutex);
+       boost::mutex::scoped_lock lock (_state_mutex);
 
-       while (_queued_full_in_memory > _maximum_frames_in_memory) {
-               /* The queue is too big; wait until that is sorted out */
+       while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
+               /* The queue is too big, and the main writer thread can run and fix it, so
+                  wake it and wait until it has done.
+               */
+               _empty_condition.notify_all ();
+               _full_condition.wait (lock);
+       }
+
+       QueueItem qi;
+       qi.type = QueueItem::Type::REPEAT;
+       qi.reel = video_reel (frame);
+       qi.frame = frame - _reels[qi.reel].start ();
+       if (film()->three_d() && eyes == Eyes::BOTH) {
+               qi.eyes = Eyes::LEFT;
+               _queue.push_back (qi);
+               qi.eyes = Eyes::RIGHT;
+               _queue.push_back (qi);
+       } else {
+               qi.eyes = eyes;
+               _queue.push_back (qi);
+       }
+
+       /* Now there's something to do: wake anything wait()ing on _empty_condition */
+       _empty_condition.notify_all ();
+}
+
+
+void
+Writer::fake_write (Frame frame, Eyes eyes)
+{
+       boost::mutex::scoped_lock lock (_state_mutex);
+
+       while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
+               /* The queue is too big, and the main writer thread can run and fix it, so
+                  wake it and wait until it has done.
+               */
+               _empty_condition.notify_all ();
                _full_condition.wait (lock);
        }
-       
-       FILE* ifi = fopen_boost (_film->info_path (frame, eyes), "r");
-       dcp::FrameInfo info (ifi);
-       fclose (ifi);
-       
+
+       size_t const reel = video_reel (frame);
+       Frame const frame_in_reel = frame - _reels[reel].start ();
+
        QueueItem qi;
-       qi.type = QueueItem::FAKE;
-       qi.size = info.size;
-       qi.frame = frame;
-       if (_film->three_d() && eyes == EYES_BOTH) {
-               qi.eyes = EYES_LEFT;
+       qi.type = QueueItem::Type::FAKE;
+
+       {
+               shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
+               qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
+       }
+
+       qi.reel = reel;
+       qi.frame = frame_in_reel;
+       if (film()->three_d() && eyes == Eyes::BOTH) {
+               qi.eyes = Eyes::LEFT;
                _queue.push_back (qi);
-               qi.eyes = EYES_RIGHT;
+               qi.eyes = Eyes::RIGHT;
                _queue.push_back (qi);
        } else {
                qi.eyes = eyes;
@@ -211,16 +255,85 @@ Writer::fake_write (int frame, Eyes eyes)
        _empty_condition.notify_all ();
 }
 
-/** This method is not thread safe */
+
+/** Write some audio frames to the DCP.
+ *  @param audio Audio data.
+ *  @param time Time of this data within the DCP.
+ *  This method is not thread safe.
+ */
+void
+Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
+{
+       DCPOMATIC_ASSERT (audio);
+
+       int const afr = film()->audio_frame_rate();
+
+       DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
+
+       /* The audio we get might span a reel boundary, and if so we have to write it in bits */
+
+       DCPTime t = time;
+       while (t < end) {
+
+               if (_audio_reel == _reels.end ()) {
+                       /* This audio is off the end of the last reel; ignore it */
+                       return;
+               }
+
+               if (end <= _audio_reel->period().to) {
+                       /* Easy case: we can write all the audio to this reel */
+                       _audio_reel->write (audio);
+                       t = end;
+               } else if (_audio_reel->period().to <= t) {
+                       /* This reel is entirely before the start of our audio; just skip the reel */
+                       ++_audio_reel;
+               } else {
+                       /* This audio is over a reel boundary; split the audio into two and write the first part */
+                       DCPTime part_lengths[2] = {
+                               _audio_reel->period().to - t,
+                               end - _audio_reel->period().to
+                       };
+
+                       /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
+                       Frame part_frames[2] = {
+                               part_lengths[0].frames_ceil(afr),
+                               part_lengths[1].frames_floor(afr)
+                       };
+
+                       DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
+
+                       if (part_frames[0]) {
+                               auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
+                               _audio_reel->write (part);
+                       }
+
+                       if (part_frames[1]) {
+                               audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
+                       } else {
+                               audio.reset ();
+                       }
+
+                       ++_audio_reel;
+                       t += part_lengths[0];
+               }
+       }
+}
+
+
 void
-Writer::write (shared_ptr<const AudioBuffers> audio)
+Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 {
-       if (_sound_mxf_writer) {
-               _sound_mxf_writer->write (audio->data(), audio->frames());
+       if (_atmos_reel->period().to == time) {
+               ++_atmos_reel;
+               DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
        }
+
+       /* We assume that we get a video frame's worth of data here */
+       _atmos_reel->write (atmos, metadata);
 }
 
-/** This must be called from Writer::thread() with an appropriate lock held */
+
+/** Caller must hold a lock on _state_mutex */
 bool
 Writer::have_sequenced_image_at_queue_head ()
 {
@@ -229,53 +342,62 @@ Writer::have_sequenced_image_at_queue_head ()
        }
 
        _queue.sort ();
+       auto const & f = _queue.front();
+       return _last_written[f.reel].next(f);
+}
 
-       /* The queue should contain only EYES_LEFT/EYES_RIGHT pairs or EYES_BOTH */
 
-       if (_queue.front().eyes == EYES_BOTH) {
+bool
+Writer::LastWritten::next (QueueItem qi) const
+{
+       if (qi.eyes == Eyes::BOTH) {
                /* 2D */
-               return _queue.front().frame == (_last_written_frame + 1);
+               return qi.frame == (_frame + 1);
        }
 
        /* 3D */
 
-       if (_last_written_eyes == EYES_LEFT && _queue.front().frame == _last_written_frame && _queue.front().eyes == EYES_RIGHT) {
+       if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
                return true;
        }
 
-       if (_last_written_eyes == EYES_RIGHT && _queue.front().frame == (_last_written_frame + 1) && _queue.front().eyes == EYES_LEFT) {
+       if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
                return true;
        }
 
        return false;
 }
 
+
+void
+Writer::LastWritten::update (QueueItem qi)
+{
+       _frame = qi.frame;
+       _eyes = qi.eyes;
+}
+
+
 void
 Writer::thread ()
 try
 {
+       start_of_thread ("Writer");
+
        while (true)
        {
-               boost::mutex::scoped_lock lock (_mutex);
-
-               /* This is for debugging only */
-               bool done_something = false;
+               boost::mutex::scoped_lock lock (_state_mutex);
 
                while (true) {
-                       
+
                        if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
                                /* We've got something to do: go and do it */
                                break;
                        }
 
                        /* Nothing to do: wait until something happens which may indicate that we do */
-                       LOG_TIMING (N_("writer sleeps with a queue of %1"), _queue.size());
+                       LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
                        _empty_condition.wait (lock);
-                       LOG_TIMING (N_("writer wakes with a queue of %1"), _queue.size());
-               }
-
-               if (_finish && _queue.empty()) {
-                       return;
+                       LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
                }
 
                /* We stop here if we have been asked to finish, and if either the queue
@@ -284,113 +406,93 @@ try
                   _finish is true).
                */
                if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
-                       done_something = true;
                        /* (Hopefully temporarily) log anything that was not written */
                        if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
                                LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
-                               for (list<QueueItem>::const_iterator i = _queue.begin(); i != _queue.end(); ++i) {
-                                       if (i->type == QueueItem::FULL) {
-                                               LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i->frame, i->eyes);
+                               for (auto const& i: _queue) {
+                                       if (i.type == QueueItem::Type::FULL) {
+                                               LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
                                        } else {
-                                               LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i->size, i->frame, i->eyes);
-                                       }                                               
+                                               LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
+                                       }
                                }
-                               LOG_WARNING (N_("Last written frame %1, last written eyes %2"), _last_written_frame, _last_written_eyes);
                        }
                        return;
                }
+
                /* Write any frames that we can write; i.e. those that are in sequence. */
                while (have_sequenced_image_at_queue_head ()) {
-                       done_something = true;
-                       QueueItem qi = _queue.front ();
+                       auto qi = _queue.front ();
+                       _last_written[qi.reel].update (qi);
                        _queue.pop_front ();
-                       if (qi.type == QueueItem::FULL && qi.encoded) {
+                       if (qi.type == QueueItem::Type::FULL && qi.encoded) {
                                --_queued_full_in_memory;
                        }
 
                        lock.unlock ();
+
+                       auto& reel = _reels[qi.reel];
+
                        switch (qi.type) {
-                       case QueueItem::FULL:
-                       {
-                               LOG_GENERAL (N_("Writer FULL-writes %1 to MXF"), qi.frame);
+                       case QueueItem::Type::FULL:
+                               LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
                                if (!qi.encoded) {
-                                       qi.encoded.reset (new EncodedData (_film->j2c_path (qi.frame, qi.eyes, false)));
+                                       qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
                                }
-
-                               dcp::FrameInfo fin = _picture_mxf_writer->write (qi.encoded->data(), qi.encoded->size());
-                               qi.encoded->write_info (_film, qi.frame, qi.eyes, fin);
-                               _last_written[qi.eyes] = qi.encoded;
+                               reel.write (qi.encoded, qi.frame, qi.eyes);
                                ++_full_written;
                                break;
-                       }
-                       case QueueItem::FAKE:
-                               LOG_GENERAL (N_("Writer FAKE-writes %1 to MXF"), qi.frame);
-                               _picture_mxf_writer->fake_write (qi.size);
-                               _last_written[qi.eyes].reset ();
+                       case QueueItem::Type::FAKE:
+                               LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
+                               reel.fake_write (qi.size);
                                ++_fake_written;
                                break;
+                       case QueueItem::Type::REPEAT:
+                               LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
+                               reel.repeat_write (qi.frame, qi.eyes);
+                               ++_repeat_written;
+                               break;
                        }
-                       lock.lock ();
 
-                       _last_written_frame = qi.frame;
-                       _last_written_eyes = qi.eyes;
-                       
-                       shared_ptr<Job> job = _job.lock ();
-                       DCPOMATIC_ASSERT (job);
-                       int64_t total = _film->length().frames (_film->video_frame_rate ());
-                       if (_film->three_d ()) {
-                               /* _full_written and so on are incremented for each eye, so we need to double the total
-                                  frames to get the correct progress.
-                               */
-                               total *= 2;
-                       }
-                       if (total) {
-                               job->set_progress (float (_full_written + _fake_written) / total);
-                       }
+                       lock.lock ();
+                       _full_condition.notify_all ();
                }
 
                while (_queued_full_in_memory > _maximum_frames_in_memory) {
-                       done_something = true;
                        /* Too many frames in memory which can't yet be written to the stream.
                           Write some FULL frames to disk.
                        */
 
                        /* Find one from the back of the queue */
                        _queue.sort ();
-                       list<QueueItem>::reverse_iterator i = _queue.rbegin ();
-                       while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
+                       auto i = _queue.rbegin ();
+                       while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
                                ++i;
                        }
 
                        DCPOMATIC_ASSERT (i != _queue.rend());
-                       QueueItem qi = *i;
-
                        ++_pushed_to_disk;
-                       
+                       /* For the log message below */
+                       int const awaiting = _last_written[_queue.front().reel].frame() + 1;
                        lock.unlock ();
 
-                       LOG_GENERAL (
-                               "Writer full (awaiting %1 [last eye was %2]); pushes %3 to disk",
-                               _last_written_frame + 1,
-                               _last_written_eyes, qi.frame
+                       /* i is valid here, even though we don't hold a lock on the mutex,
+                          since list iterators are unaffected by insertion and only this
+                          thread could erase the last item in the list.
+                       */
+
+                       LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
+
+                       i->encoded->write_via_temp (
+                               film()->j2c_path(i->reel, i->frame, i->eyes, true),
+                               film()->j2c_path(i->reel, i->frame, i->eyes, false)
                                );
-                       
-                       qi.encoded->write (_film, qi.frame, qi.eyes);
+
                        lock.lock ();
-                       qi.encoded.reset ();
+                       i->encoded.reset ();
                        --_queued_full_in_memory;
+                       _full_condition.notify_all ();
                }
-
-               if (!done_something) {
-                       LOG_DEBUG_NC ("Writer loop ran without doing anything");
-                       LOG_DEBUG ("_queued_full_in_memory=%1", _queued_full_in_memory);
-                       LOG_DEBUG ("_queue_size=%1", _queue.size ());
-                       LOG_DEBUG ("_finish=%1", _finish);
-                       LOG_DEBUG ("_last_written_frame=%1", _last_written_frame);
-               }
-
-               /* The queue has probably just gone down a bit; notify anything wait()ing on _full_condition */
-               _full_condition.notify_all ();
        }
 }
 catch (...)
@@ -398,290 +500,445 @@ catch (...)
        store_current ();
 }
 
+
 void
 Writer::terminate_thread (bool can_throw)
 {
-       boost::mutex::scoped_lock lock (_mutex);
-       if (_thread == 0) {
-               return;
-       }
-       
+       boost::this_thread::disable_interruption dis;
+
+       boost::mutex::scoped_lock lock (_state_mutex);
+
        _finish = true;
        _empty_condition.notify_all ();
        _full_condition.notify_all ();
        lock.unlock ();
 
-       _thread->join ();
+       try {
+               _thread.join ();
+       } catch (...) {}
+
        if (can_throw) {
                rethrow ();
        }
-       
-       delete _thread;
-       _thread = 0;
-}      
+}
+
 
 void
-Writer::finish ()
+Writer::calculate_digests ()
 {
-       if (!_thread) {
-               return;
+       auto job = _job.lock ();
+       if (job) {
+               job->sub (_("Computing digests"));
        }
-       
-       terminate_thread (true);
-
-       _picture_mxf_writer->finalize ();
-       if (_sound_mxf_writer) {
-               _sound_mxf_writer->finalize ();
-       }
-       
-       /* Hard-link the video MXF into the DCP */
-       boost::filesystem::path video_from;
-       video_from /= _film->internal_video_mxf_dir();
-       video_from /= _film->internal_video_mxf_filename();
-       
-       boost::filesystem::path video_to;
-       video_to /= _film->dir (_film->dcp_name());
-       video_to /= _film->video_mxf_filename ();
-
-       boost::system::error_code ec;
-       boost::filesystem::create_hard_link (video_from, video_to, ec);
-       if (ec) {
-               LOG_WARNING_NC ("Hard-link failed; copying instead");
-               boost::filesystem::copy_file (video_from, video_to, ec);
-               if (ec) {
-                       LOG_ERROR ("Failed to copy video file from %1 to %2 (%3)", video_from.string(), video_to.string(), ec.message ());
-                       throw FileError (ec.message(), video_from);
-               }
+
+       boost::asio::io_service service;
+       boost::thread_group pool;
+
+       auto work = make_shared<boost::asio::io_service::work>(service);
+
+       int const threads = max (1, Config::instance()->master_encoding_threads());
+
+       for (int i = 0; i < threads; ++i) {
+               pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
+       }
+
+       std::function<void (float)> set_progress;
+       if (job) {
+               set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
+       } else {
+               set_progress = [](float) {
+                       boost::this_thread::interruption_point();
+               };
        }
 
-       _picture_mxf->set_file (video_to);
+       for (auto& i: _reels) {
+               service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
+       }
+       service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
+
+       work.reset ();
+
+       try {
+               pool.join_all ();
+       } catch (boost::thread_interrupted) {
+               /* join_all was interrupted, so we need to interrupt the threads
+                * in our pool then try again to join them.
+                */
+               pool.interrupt_all ();
+               pool.join_all ();
+       }
 
-       /* Move the audio MXF into the DCP */
+       service.stop ();
+}
 
-       if (_sound_mxf) {
-               boost::filesystem::path audio_to;
-               audio_to /= _film->dir (_film->dcp_name ());
-               audio_to /= _film->audio_mxf_filename ();
-               
-               boost::filesystem::rename (_film->file (_film->audio_mxf_filename ()), audio_to, ec);
-               if (ec) {
-                       throw FileError (
-                               String::compose (_("could not move audio MXF into the DCP (%1)"), ec.value ()), _film->file (_film->audio_mxf_filename ())
-                               );
-               }
 
-               _sound_mxf->set_file (audio_to);
+/** @param output_dcp Path to DCP folder to write */
+void
+Writer::finish (boost::filesystem::path output_dcp)
+{
+       if (_thread.joinable()) {
+               LOG_GENERAL_NC ("Terminating writer thread");
+               terminate_thread (true);
+       }
+
+       LOG_GENERAL_NC ("Finishing ReelWriters");
+
+       for (auto& i: _reels) {
+               write_hanging_text (i);
+               i.finish (output_dcp);
        }
 
-       dcp::DCP dcp (_film->dir (_film->dcp_name()));
+       LOG_GENERAL_NC ("Writing XML");
+
+       dcp::DCP dcp (output_dcp);
 
-       shared_ptr<dcp::CPL> cpl (
-               new dcp::CPL (
-                       _film->dcp_name(),
-                       _film->dcp_content_type()->libdcp_kind ()
-                       )
+       auto cpl = make_shared<dcp::CPL>(
+               film()->dcp_name(),
+               film()->dcp_content_type()->libdcp_kind(),
+               film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
                );
-       
+
        dcp.add (cpl);
 
-       shared_ptr<dcp::Reel> reel (new dcp::Reel ());
+       calculate_digests ();
 
-       shared_ptr<dcp::MonoPictureMXF> mono = dynamic_pointer_cast<dcp::MonoPictureMXF> (_picture_mxf);
-       if (mono) {
-               reel->add (shared_ptr<dcp::ReelPictureAsset> (new dcp::ReelMonoPictureAsset (mono, 0)));
-               dcp.add (mono);
-       }
+       /* Add reels */
 
-       shared_ptr<dcp::StereoPictureMXF> stereo = dynamic_pointer_cast<dcp::StereoPictureMXF> (_picture_mxf);
-       if (stereo) {
-               reel->add (shared_ptr<dcp::ReelPictureAsset> (new dcp::ReelStereoPictureAsset (stereo, 0)));
-               dcp.add (stereo);
+       for (auto& i: _reels) {
+               cpl->add (i.create_reel(_reel_assets, _fonts, _chosen_interop_font, output_dcp, _have_subtitles, _have_closed_captions));
        }
 
-       if (_sound_mxf) {
-               reel->add (shared_ptr<dcp::ReelSoundAsset> (new dcp::ReelSoundAsset (_sound_mxf, 0)));
-               dcp.add (_sound_mxf);
+       /* Add metadata */
+
+       auto creator = Config::instance()->dcp_creator();
+       if (creator.empty()) {
+               creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
        }
 
-       if (_subtitle_content) {
-               _subtitle_content->write_xml (_film->dir (_film->dcp_name ()) / _film->subtitle_xml_filename ());
-               reel->add (shared_ptr<dcp::ReelSubtitleAsset> (
-                                  new dcp::ReelSubtitleAsset (
-                                          _subtitle_content,
-                                          dcp::Fraction (_film->video_frame_rate(), 1),
-                                          _picture_mxf->intrinsic_duration (),
-                                          0
-                                          )
-                                  ));
-               
-               dcp.add (_subtitle_content);
+       auto issuer = Config::instance()->dcp_issuer();
+       if (issuer.empty()) {
+               issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
        }
-       
-       cpl->add (reel);
 
-       shared_ptr<Job> job = _job.lock ();
-       DCPOMATIC_ASSERT (job);
+       cpl->set_creator (creator);
+       cpl->set_issuer (issuer);
 
-       job->sub (_("Computing image digest"));
-       _picture_mxf->hash (boost::bind (&Job::set_progress, job.get(), _1, false));
+       cpl->set_ratings (film()->ratings());
 
-       if (_sound_mxf) {
-               job->sub (_("Computing audio digest"));
-               _sound_mxf->hash (boost::bind (&Job::set_progress, job.get(), _1, false));
+       vector<dcp::ContentVersion> cv;
+       for (auto i: film()->content_versions()) {
+               cv.push_back (dcp::ContentVersion(i));
+       }
+       if (cv.empty()) {
+               cv = { dcp::ContentVersion("1") };
+       }
+       cpl->set_content_versions (cv);
+
+       cpl->set_full_content_title_text (film()->name());
+       cpl->set_full_content_title_text_language (film()->name_language());
+       if (film()->release_territory()) {
+               cpl->set_release_territory (*film()->release_territory());
+       }
+       cpl->set_version_number (film()->version_number());
+       cpl->set_status (film()->status());
+       if (film()->chain()) {
+               cpl->set_chain (*film()->chain());
+       }
+       if (film()->distributor()) {
+               cpl->set_distributor (*film()->distributor());
+       }
+       if (film()->facility()) {
+               cpl->set_facility (*film()->facility());
+       }
+       if (film()->luminance()) {
+               cpl->set_luminance (*film()->luminance());
+       }
+       if (film()->sign_language_video_language()) {
+               cpl->set_sign_language_video_language (*film()->sign_language_video_language());
        }
 
-       dcp::XMLMetadata meta;
-       meta.issuer = Config::instance()->dcp_issuer ();
-       meta.creator = String::compose ("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
-       meta.set_issue_date_now ();
+       auto ac = film()->mapped_audio_channels();
+       dcp::MCASoundField field = (
+               find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSL)) != ac.end() ||
+               find(ac.begin(), ac.end(), static_cast<int>(dcp::Channel::BSR)) != ac.end()
+               ) ? dcp::MCASoundField::SEVEN_POINT_ONE : dcp::MCASoundField::FIVE_POINT_ONE;
 
-       shared_ptr<const dcp::Signer> signer;
-       if (_film->is_signed ()) {
-               signer = Config::instance()->signer ();
-               /* We did check earlier, but check again here to be on the safe side */
-               if (!signer->valid ()) {
-                       throw InvalidSignerError ();
+       dcp::MainSoundConfiguration msc (field, film()->audio_channels());
+       for (auto i: ac) {
+               if (static_cast<int>(i) < film()->audio_channels()) {
+                       msc.set_mapping (i, static_cast<dcp::Channel>(i));
                }
        }
 
-       dcp.write_xml (_film->interop () ? dcp::INTEROP : dcp::SMPTE, meta, signer);
+       cpl->set_main_sound_configuration (msc.to_string());
+       cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
+       cpl->set_main_picture_stored_area (film()->frame_size());
 
-       LOG_GENERAL (
-               N_("Wrote %1 FULL, %2 FAKE, %3 pushed to disk"), _full_written, _fake_written, _pushed_to_disk
-               );
-}
-
-bool
-Writer::check_existing_picture_mxf_frame (FILE* mxf, int f, Eyes eyes)
-{
-       /* Read the frame info as written */
-       FILE* ifi = fopen_boost (_film->info_path (f, eyes), "r");
-       if (!ifi) {
-               LOG_GENERAL ("Existing frame %1 has no info file", f);
-               return false;
+       auto active_area = film()->active_area();
+       if (active_area.width > 0 && active_area.height > 0) {
+               /* It's not allowed to have a zero active area width or height */
+               cpl->set_main_picture_active_area (active_area);
        }
-       
-       dcp::FrameInfo info (ifi);
-       fclose (ifi);
-       if (info.size == 0) {
-               LOG_GENERAL ("Existing frame %1 has no info file", f);
-               return false;
-       }
-       
-       /* Read the data from the MXF and hash it */
-       dcpomatic_fseek (mxf, info.offset, SEEK_SET);
-       EncodedData data (info.size);
-       size_t const read = fread (data.data(), 1, data.size(), mxf);
-       if (read != static_cast<size_t> (data.size ())) {
-               LOG_GENERAL ("Existing frame %1 is incomplete", f);
-               return false;
+
+       auto sl = film()->subtitle_languages().second;
+       if (!sl.empty()) {
+               cpl->set_additional_subtitle_languages(sl);
        }
 
-       MD5Digester digester;
-       digester.add (data.data(), data.size());
-       if (digester.get() != info.hash) {
-               LOG_GENERAL ("Existing frame %1 failed hash check", f);
-               return false;
+       auto signer = Config::instance()->signer_chain();
+       /* We did check earlier, but check again here to be on the safe side */
+       string reason;
+       if (!signer->valid (&reason)) {
+               throw InvalidSignerError (reason);
        }
 
-       return true;
+       dcp.set_issuer(issuer);
+       dcp.set_creator(creator);
+       dcp.set_annotation_text(film()->dcp_name());
+
+       dcp.write_xml (signer, Config::instance()->dcp_metadata_filename_format());
+
+       LOG_GENERAL (
+               N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
+               );
+
+       write_cover_sheet (output_dcp);
 }
 
+
 void
-Writer::check_existing_picture_mxf ()
+Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 {
-       /* Try to open the existing MXF */
-       boost::filesystem::path p;
-       p /= _film->internal_video_mxf_dir ();
-       p /= _film->internal_video_mxf_filename ();
-       FILE* mxf = fopen_boost (p, "rb");
-       if (!mxf) {
-               LOG_GENERAL ("Could not open existing MXF at %1 (errno=%2)", p.string(), errno);
-               return;
+       auto const cover = film()->file("COVER_SHEET.txt");
+       dcp::File f(cover, "w");
+       if (!f) {
+               throw OpenFileError (cover, errno, OpenFileError::WRITE);
        }
 
-       int N = 0;
-       for (boost::filesystem::directory_iterator i (_film->info_dir ()); i != boost::filesystem::directory_iterator (); ++i) {
-               ++N;
+       auto text = Config::instance()->cover_sheet ();
+       boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
+       auto cpls = film()->cpls();
+       if (!cpls.empty()) {
+               boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
        }
+       boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
+       boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 
-       while (true) {
+       auto audio_language = film()->audio_language();
+       if (audio_language) {
+               boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
+       } else {
+               boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
+       }
 
-               shared_ptr<Job> job = _job.lock ();
-               DCPOMATIC_ASSERT (job);
+       auto subtitle_languages = film()->subtitle_languages();
+       if (subtitle_languages.first) {
+               boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
+       } else {
+               boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
+       }
 
-               if (N > 0) {
-                       job->set_progress (float (_first_nonexistant_frame) / N);
+       boost::uintmax_t size = 0;
+       for (
+               auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
+               i != boost::filesystem::recursive_directory_iterator();
+               ++i) {
+               if (boost::filesystem::is_regular_file (i->path())) {
+                       size += boost::filesystem::file_size (i->path());
                }
+       }
 
-               if (_film->three_d ()) {
-                       if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_LEFT)) {
-                               break;
-                       }
-                       if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_RIGHT)) {
-                               break;
-                       }
-               } else {
-                       if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_BOTH)) {
-                               break;
-                       }
-               }
+       if (size > (1000000000L)) {
+               boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
+       } else {
+               boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
+       }
+
+       auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
+       auto description = String::compose("%1.%2", ch.first, ch.second);
 
-               LOG_GENERAL ("Have existing frame %1", _first_nonexistant_frame);
-               ++_first_nonexistant_frame;
+       if (description == "0.0") {
+               description = _("None");
+       } else if (description == "1.0") {
+               description = _("Mono");
+       } else if (description == "2.0") {
+               description = _("Stereo");
        }
+       boost::algorithm::replace_all (text, "$AUDIO", description);
+
+       auto const hmsf = film()->length().split(film()->video_frame_rate());
+       string length;
+       if (hmsf.h == 0 && hmsf.m == 0) {
+               length = String::compose("%1s", hmsf.s);
+       } else if (hmsf.h == 0 && hmsf.m > 0) {
+               length = String::compose("%1m%2s", hmsf.m, hmsf.s);
+       } else if (hmsf.h > 0 && hmsf.m > 0) {
+               length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
+       }
+
+       boost::algorithm::replace_all (text, "$LENGTH", length);
 
-       fclose (mxf);
+       f.checked_write(text.c_str(), text.length());
 }
 
-/** @param frame Frame index.
+
+/** @param frame Frame index within the whole DCP.
  *  @return true if we can fake-write this frame.
  */
 bool
-Writer::can_fake_write (int frame) const
+Writer::can_fake_write (Frame frame) const
 {
+       if (film()->encrypted()) {
+               /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
+               return false;
+       }
+
        /* We have to do a proper write of the first frame so that we can set up the JPEG2000
-          parameters in the MXF writer.
+          parameters in the asset writer.
        */
-       return (frame != 0 && frame < _first_nonexistant_frame);
+
+       auto const & reel = _reels[video_reel(frame)];
+
+       /* Make frame relative to the start of the reel */
+       frame -= reel.start ();
+       return (frame != 0 && frame < reel.first_nonexistent_frame());
 }
 
+
+/** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 void
-Writer::write (PlayerSubtitles subs)
+Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 {
-       if (subs.text.empty ()) {
-               return;
+       vector<ReelWriter>::iterator* reel = nullptr;
+
+       switch (type) {
+       case TextType::OPEN_SUBTITLE:
+               reel = &_subtitle_reel;
+               _have_subtitles = true;
+               break;
+       case TextType::CLOSED_CAPTION:
+               DCPOMATIC_ASSERT (track);
+               DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
+               reel = &_caption_reels[*track];
+               _have_closed_captions.insert (*track);
+               break;
+       default:
+               DCPOMATIC_ASSERT (false);
        }
 
-       if (!_subtitle_content) {
-               _subtitle_content.reset (new dcp::InteropSubtitleContent (_film->name(), _film->subtitle_language ()));
+       DCPOMATIC_ASSERT (*reel != _reels.end());
+       while ((*reel)->period().to <= period.from) {
+               ++(*reel);
+               DCPOMATIC_ASSERT (*reel != _reels.end());
+               write_hanging_text (**reel);
        }
-       
-       for (list<dcp::SubtitleString>::const_iterator i = subs.text.begin(); i != subs.text.end(); ++i) {
-               _subtitle_content->add (*i);
+
+       auto back_off = [this](DCPTimePeriod period) {
+               period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
+               return period;
+       };
+
+       if (period.to > (*reel)->period().to) {
+               /* This text goes off the end of the reel.  Store parts of it that should go into
+                * other reels.
+                */
+               for (auto i = std::next(*reel); i != _reels.end(); ++i) {
+                       auto overlap = i->period().overlap(period);
+                       if (overlap) {
+                               _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
+                       }
+               }
+               /* Back off from the reel boundary by a couple of frames to avoid tripping checks
+                * for subtitles being too close together.
+                */
+               period.to = (*reel)->period().to;
+               period = back_off(period);
        }
+
+       (*reel)->write(text, type, track, period, _fonts);
 }
 
+
 void
-Writer::write (list<shared_ptr<Font> > fonts)
+Writer::write (vector<shared_ptr<Font>> fonts)
 {
-       if (fonts.empty ()) {
+       if (fonts.empty()) {
                return;
        }
-       
-       if (!_subtitle_content) {
-               _subtitle_content.reset (new dcp::InteropSubtitleContent (_film->name(), _film->subtitle_language ()));
-       }
-       
-       for (list<shared_ptr<Font> >::const_iterator i = fonts.begin(); i != fonts.end(); ++i) {
-               /* XXX: this LiberationSans-Regular needs to be a path to a DCP-o-matic-distributed copy */
-               _subtitle_content->add_font ((*i)->id, (*i)->file.get_value_or ("LiberationSans-Regular.ttf").leaf().string ());
+
+       /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
+       auto fix_id = [](string id) {
+               return id.empty() ? "font" : id;
+       };
+
+       if (film()->interop()) {
+               /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
+                * even write them as they upset some validators.  Set up _fonts so that every
+                * font used by any subtitle will be written with the same ID.
+                */
+               for (size_t i = 0; i < fonts.size(); ++i) {
+                       _fonts.put(fonts[i], fix_id(fonts[0]->id()));
+               }
+               _chosen_interop_font = fonts[0];
+       } else {
+               set<string> used_ids;
+
+               /* Return the index of a _N at the end of a string, or string::npos */
+               auto underscore_number_position = [](string s) {
+                       auto last_underscore = s.find_last_of("_");
+                       if (last_underscore == string::npos) {
+                               return string::npos;
+                       }
+
+                       for (auto i = last_underscore + 1; i < s.size(); ++i) {
+                               if (!isdigit(s[i])) {
+                                       return string::npos;
+                               }
+                       }
+
+                       return last_underscore;
+               };
+
+               /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
+               for (auto font: fonts) {
+                       auto id = fix_id(font->id());
+                       if (used_ids.find(id) == used_ids.end()) {
+                               /* This ID is unique so we can just use it as-is */
+                               _fonts.put(font, id);
+                               used_ids.insert(id);
+                       } else {
+                               auto end = underscore_number_position(id);
+                               if (end == string::npos) {
+                                       /* This string has no _N suffix, so add one */
+                                       id += "_0";
+                                       end = underscore_number_position(id);
+                               }
+
+                               ++end;
+
+                               /* Increment the suffix until we find a unique one */
+                               auto number = dcp::raw_convert<int>(id.substr(end));
+                               while (used_ids.find(id) != used_ids.end()) {
+                                       ++number;
+                                       id = String::compose("%1_%2", id.substr(0, end - 1), number);
+                               }
+                               used_ids.insert(id);
+                       }
+                       _fonts.put(font, id);
+               }
+
+               DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
        }
 }
 
+
 bool
 operator< (QueueItem const & a, QueueItem const & b)
 {
+       if (a.reel != b.reel) {
+               return a.reel < b.reel;
+       }
+
        if (a.frame != b.frame) {
                return a.frame < b.frame;
        }
@@ -689,8 +946,93 @@ operator< (QueueItem const & a, QueueItem const & b)
        return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 }
 
+
 bool
 operator== (QueueItem const & a, QueueItem const & b)
 {
-       return a.frame == b.frame && a.eyes == b.eyes;
+       return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
+}
+
+
+void
+Writer::set_encoder_threads (int threads)
+{
+       boost::mutex::scoped_lock lm (_state_mutex);
+       _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
+       _maximum_queue_size = threads * 16;
+}
+
+
+void
+Writer::write (ReferencedReelAsset asset)
+{
+       _reel_assets.push_back (asset);
+}
+
+
+size_t
+Writer::video_reel (int frame) const
+{
+       auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
+       size_t i = 0;
+       while (i < _reels.size() && !_reels[i].period().contains (t)) {
+               ++i;
+       }
+
+       DCPOMATIC_ASSERT (i < _reels.size ());
+       return i;
+}
+
+
+void
+Writer::set_digest_progress (Job* job, float progress)
+{
+       boost::mutex::scoped_lock lm (_digest_progresses_mutex);
+
+       _digest_progresses[boost::this_thread::get_id()] = progress;
+       float min_progress = FLT_MAX;
+       for (auto const& i: _digest_progresses) {
+               min_progress = min (min_progress, i.second);
+       }
+
+       job->set_progress (min_progress);
+
+       Waker waker;
+       waker.nudge ();
+
+       boost::this_thread::interruption_point();
+}
+
+
+/** Calculate hashes for any referenced MXF assets which do not already have one */
+void
+Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
+try
+{
+       for (auto const& i: _reel_assets) {
+               auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
+               if (file && !file->hash()) {
+                       file->asset_ref().asset()->hash (set_progress);
+                       file->set_hash (file->asset_ref().asset()->hash());
+               }
+       }
+} catch (boost::thread_interrupted) {
+       /* set_progress contains an interruption_point, so any of these methods
+        * may throw thread_interrupted, at which point we just give up.
+        */
+}
+
+
+void
+Writer::write_hanging_text (ReelWriter& reel)
+{
+       vector<HangingText> new_hanging_texts;
+       for (auto i: _hanging_texts) {
+               if (i.period.from == reel.period().from) {
+                       reel.write (i.text, i.type, i.track, i.period, _fonts);
+               } else {
+                       new_hanging_texts.push_back (i);
+               }
+       }
+       _hanging_texts = new_hanging_texts;
 }