src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "cross.h"
  27 #include "dcp_content_type.h"
  28 #include "dcp_video.h"
  29 #include "dcpomatic_log.h"
  30 #include "film.h"
  31 #include "job.h"
  32 #include "log.h"
  33 #include "ratio.h"
  34 #include "reel_writer.h"
  35 #include "text_content.h"
  36 #include "util.h"
  37 #include "version.h"
  38 #include "writer.h"
  39 #include <dcp/cpl.h>
  40 #include <dcp/locale_convert.h>
  41 #include <dcp/raw_convert.h>
  42 #include <dcp/reel_file_asset.h>
  43 #include <cerrno>
  44 #include <cfloat>
  45 #include <set>
  46
  47 #include "i18n.h"
  48
  49
  50 /* OS X strikes again */
  51 #undef set_key
  52
  53
  54 using std::cout;
  55 using std::dynamic_pointer_cast;
  56 using std::make_shared;
  57 using std::max;
  58 using std::min;
  59 using std::shared_ptr;
  60 using std::set;
  61 using std::string;
  62 using std::vector;
  63 using std::weak_ptr;
  64 using boost::optional;
  65 #if BOOST_VERSION >= 106100
  66 using namespace boost::placeholders;
  67 #endif
  68 using dcp::Data;
  69 using dcp::ArrayData;
  70 using namespace dcpomatic;
  71
  72
  73 /** @param j Job to report progress to, or 0.
  74  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  75  */
  76 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
  77         : WeakConstFilm (weak_film)
  78         , _job (j)
  79         /* These will be reset to sensible values when J2KEncoder is created */
  80         , _maximum_frames_in_memory (8)
  81         , _maximum_queue_size (8)
  82         , _text_only (text_only)
  83 {
  84         auto job = _job.lock ();
  85
  86         int reel_index = 0;
  87         auto const reels = film()->reels();
  88         for (auto p: reels) {
  89                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  90         }
  91
  92         _last_written.resize (reels.size());
  93
  94         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  95            and captions arrive to the Writer in sequence.  This is not so for video.
  96         */
  97         _audio_reel = _reels.begin ();
  98         _subtitle_reel = _reels.begin ();
  99         for (auto i: film()->closed_caption_tracks()) {
 100                 _caption_reels[i] = _reels.begin ();
 101         }
 102         _atmos_reel = _reels.begin ();
 103
 104         /* Check that the signer is OK */
 105         string reason;
 106         if (!Config::instance()->signer_chain()->valid(&reason)) {
 107                 throw InvalidSignerError (reason);
 108         }
 109 }
 110
 111
 112 void
 113 Writer::start ()
 114 {
 115         if (!_text_only) {
 116                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 117 #ifdef DCPOMATIC_LINUX
 118                 pthread_setname_np (_thread.native_handle(), "writer");
 119 #endif
 120         }
 121 }
 122
 123
 124 Writer::~Writer ()
 125 {
 126         if (!_text_only) {
 127                 terminate_thread (false);
 128         }
 129 }
 130
 131
 132 /** Pass a video frame to the writer for writing to disk at some point.
 133  *  This method can be called with frames out of order.
 134  *  @param encoded JPEG2000-encoded data.
 135  *  @param frame Frame index within the DCP.
 136  *  @param eyes Eyes that this frame image is for.
 137  */
 138 void
 139 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 140 {
 141         boost::mutex::scoped_lock lock (_state_mutex);
 142
 143         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 144                 /* There are too many full frames in memory; wake the main writer thread and
 145                    wait until it sorts everything out */
 146                 _empty_condition.notify_all ();
 147                 _full_condition.wait (lock);
 148         }
 149
 150         QueueItem qi;
 151         qi.type = QueueItem::Type::FULL;
 152         qi.encoded = encoded;
 153         qi.reel = video_reel (frame);
 154         qi.frame = frame - _reels[qi.reel].start ();
 155
 156         if (film()->three_d() && eyes == Eyes::BOTH) {
 157                 /* 2D material in a 3D DCP; fake the 3D */
 158                 qi.eyes = Eyes::LEFT;
 159                 _queue.push_back (qi);
 160                 ++_queued_full_in_memory;
 161                 qi.eyes = Eyes::RIGHT;
 162                 _queue.push_back (qi);
 163                 ++_queued_full_in_memory;
 164         } else {
 165                 qi.eyes = eyes;
 166                 _queue.push_back (qi);
 167                 ++_queued_full_in_memory;
 168         }
 169
 170         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 171         _empty_condition.notify_all ();
 172 }
 173
 174
 175 bool
 176 Writer::can_repeat (Frame frame) const
 177 {
 178         return frame > _reels[video_reel(frame)].start();
 179 }
 180
 181
 182 /** Repeat the last frame that was written to a reel as a new frame.
 183  *  @param frame Frame index within the DCP of the new (repeated) frame.
 184  *  @param eyes Eyes that this repeated frame image is for.
 185  */
 186 void
 187 Writer::repeat (Frame frame, Eyes eyes)
 188 {
 189         boost::mutex::scoped_lock lock (_state_mutex);
 190
 191         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 192                 /* The queue is too big, and the main writer thread can run and fix it, so
 193                    wake it and wait until it has done.
 194                 */
 195                 _empty_condition.notify_all ();
 196                 _full_condition.wait (lock);
 197         }
 198
 199         QueueItem qi;
 200         qi.type = QueueItem::Type::REPEAT;
 201         qi.reel = video_reel (frame);
 202         qi.frame = frame - _reels[qi.reel].start ();
 203         if (film()->three_d() && eyes == Eyes::BOTH) {
 204                 qi.eyes = Eyes::LEFT;
 205                 _queue.push_back (qi);
 206                 qi.eyes = Eyes::RIGHT;
 207                 _queue.push_back (qi);
 208         } else {
 209                 qi.eyes = eyes;
 210                 _queue.push_back (qi);
 211         }
 212
 213         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 214         _empty_condition.notify_all ();
 215 }
 216
 217
 218 void
 219 Writer::fake_write (Frame frame, Eyes eyes)
 220 {
 221         boost::mutex::scoped_lock lock (_state_mutex);
 222
 223         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 224                 /* The queue is too big, and the main writer thread can run and fix it, so
 225                    wake it and wait until it has done.
 226                 */
 227                 _empty_condition.notify_all ();
 228                 _full_condition.wait (lock);
 229         }
 230
 231         size_t const reel = video_reel (frame);
 232         Frame const frame_in_reel = frame - _reels[reel].start ();
 233
 234         QueueItem qi;
 235         qi.type = QueueItem::Type::FAKE;
 236
 237         {
 238                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 239                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 240         }
 241
 242         qi.reel = reel;
 243         qi.frame = frame_in_reel;
 244         if (film()->three_d() && eyes == Eyes::BOTH) {
 245                 qi.eyes = Eyes::LEFT;
 246                 _queue.push_back (qi);
 247                 qi.eyes = Eyes::RIGHT;
 248                 _queue.push_back (qi);
 249         } else {
 250                 qi.eyes = eyes;
 251                 _queue.push_back (qi);
 252         }
 253
 254         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 255         _empty_condition.notify_all ();
 256 }
 257
 258
 259 /** Write some audio frames to the DCP.
 260  *  @param audio Audio data.
 261  *  @param time Time of this data within the DCP.
 262  *  This method is not thread safe.
 263  */
 264 void
 265 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 266 {
 267         DCPOMATIC_ASSERT (audio);
 268
 269         int const afr = film()->audio_frame_rate();
 270
 271         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 272
 273         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 274
 275         DCPTime t = time;
 276         while (t < end) {
 277
 278                 if (_audio_reel == _reels.end ()) {
 279                         /* This audio is off the end of the last reel; ignore it */
 280                         return;
 281                 }
 282
 283                 if (end <= _audio_reel->period().to) {
 284                         /* Easy case: we can write all the audio to this reel */
 285                         _audio_reel->write (audio);
 286                         t = end;
 287                 } else if (_audio_reel->period().to <= t) {
 288                         /* This reel is entirely before the start of our audio; just skip the reel */
 289                         ++_audio_reel;
 290                 } else {
 291                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 292                         DCPTime part_lengths[2] = {
 293                                 _audio_reel->period().to - t,
 294                                 end - _audio_reel->period().to
 295                         };
 296
 297                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 298                         Frame part_frames[2] = {
 299                                 part_lengths[0].frames_ceil(afr),
 300                                 part_lengths[1].frames_floor(afr)
 301                         };
 302
 303                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 304
 305                         if (part_frames[0]) {
 306                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 307                                 _audio_reel->write (part);
 308                         }
 309
 310                         if (part_frames[1]) {
 311                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 312                         } else {
 313                                 audio.reset ();
 314                         }
 315
 316                         ++_audio_reel;
 317                         t += part_lengths[0];
 318                 }
 319         }
 320 }
 321
 322
 323 void
 324 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 325 {
 326         if (_atmos_reel->period().to == time) {
 327                 ++_atmos_reel;
 328                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 329         }
 330
 331         /* We assume that we get a video frame's worth of data here */
 332         _atmos_reel->write (atmos, metadata);
 333 }
 334
 335
 336 /** Caller must hold a lock on _state_mutex */
 337 bool
 338 Writer::have_sequenced_image_at_queue_head ()
 339 {
 340         if (_queue.empty ()) {
 341                 return false;
 342         }
 343
 344         _queue.sort ();
 345         auto const & f = _queue.front();
 346         return _last_written[f.reel].next(f);
 347 }
 348
 349
 350 bool
 351 Writer::LastWritten::next (QueueItem qi) const
 352 {
 353         if (qi.eyes == Eyes::BOTH) {
 354                 /* 2D */
 355                 return qi.frame == (_frame + 1);
 356         }
 357
 358         /* 3D */
 359
 360         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 361                 return true;
 362         }
 363
 364         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 365                 return true;
 366         }
 367
 368         return false;
 369 }
 370
 371
 372 void
 373 Writer::LastWritten::update (QueueItem qi)
 374 {
 375         _frame = qi.frame;
 376         _eyes = qi.eyes;
 377 }
 378
 379
 380 void
 381 Writer::thread ()
 382 try
 383 {
 384         start_of_thread ("Writer");
 385
 386         while (true)
 387         {
 388                 boost::mutex::scoped_lock lock (_state_mutex);
 389
 390                 while (true) {
 391
 392                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 393                                 /* We've got something to do: go and do it */
 394                                 break;
 395                         }
 396
 397                         /* Nothing to do: wait until something happens which may indicate that we do */
 398                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 399                         _empty_condition.wait (lock);
 400                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 401                 }
 402
 403                 /* We stop here if we have been asked to finish, and if either the queue
 404                    is empty or we do not have a sequenced image at its head (if this is the
 405                    case we will never terminate as no new frames will be sent once
 406                    _finish is true).
 407                 */
 408                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 409                         /* (Hopefully temporarily) log anything that was not written */
 410                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 411                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 412                                 for (auto const& i: _queue) {
 413                                         if (i.type == QueueItem::Type::FULL) {
 414                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 415                                         } else {
 416                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 417                                         }
 418                                 }
 419                         }
 420                         return;
 421                 }
 422
 423                 /* Write any frames that we can write; i.e. those that are in sequence. */
 424                 while (have_sequenced_image_at_queue_head ()) {
 425                         auto qi = _queue.front ();
 426                         _last_written[qi.reel].update (qi);
 427                         _queue.pop_front ();
 428                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 429                                 --_queued_full_in_memory;
 430                         }
 431
 432                         lock.unlock ();
 433
 434                         auto& reel = _reels[qi.reel];
 435
 436                         switch (qi.type) {
 437                         case QueueItem::Type::FULL:
 438                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 439                                 if (!qi.encoded) {
 440                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 441                                 }
 442                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 443                                 ++_full_written;
 444                                 break;
 445                         case QueueItem::Type::FAKE:
 446                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 447                                 reel.fake_write (qi.size);
 448                                 ++_fake_written;
 449                                 break;
 450                         case QueueItem::Type::REPEAT:
 451                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 452                                 reel.repeat_write (qi.frame, qi.eyes);
 453                                 ++_repeat_written;
 454                                 break;
 455                         }
 456
 457                         lock.lock ();
 458                         _full_condition.notify_all ();
 459                 }
 460
 461                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 462                         /* Too many frames in memory which can't yet be written to the stream.
 463                            Write some FULL frames to disk.
 464                         */
 465
 466                         /* Find one from the back of the queue */
 467                         _queue.sort ();
 468                         auto i = _queue.rbegin ();
 469                         while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
 470                                 ++i;
 471                         }
 472
 473                         DCPOMATIC_ASSERT (i != _queue.rend());
 474                         ++_pushed_to_disk;
 475                         /* For the log message below */
 476                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 477                         lock.unlock ();
 478
 479                         /* i is valid here, even though we don't hold a lock on the mutex,
 480                            since list iterators are unaffected by insertion and only this
 481                            thread could erase the last item in the list.
 482                         */
 483
 484                         LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
 485
 486                         i->encoded->write_via_temp (
 487                                 film()->j2c_path(i->reel, i->frame, i->eyes, true),
 488                                 film()->j2c_path(i->reel, i->frame, i->eyes, false)
 489                                 );
 490
 491                         lock.lock ();
 492                         i->encoded.reset ();
 493                         --_queued_full_in_memory;
 494                         _full_condition.notify_all ();
 495                 }
 496         }
 497 }
 498 catch (...)
 499 {
 500         store_current ();
 501 }
 502
 503
 504 void
 505 Writer::terminate_thread (bool can_throw)
 506 {
 507         boost::this_thread::disable_interruption dis;
 508
 509         boost::mutex::scoped_lock lock (_state_mutex);
 510
 511         _finish = true;
 512         _empty_condition.notify_all ();
 513         _full_condition.notify_all ();
 514         lock.unlock ();
 515
 516         try {
 517                 _thread.join ();
 518         } catch (...) {}
 519
 520         if (can_throw) {
 521                 rethrow ();
 522         }
 523 }
 524
 525
 526 void
 527 Writer::calculate_digests ()
 528 {
 529         auto job = _job.lock ();
 530         if (job) {
 531                 job->sub (_("Computing digests"));
 532         }
 533
 534         boost::asio::io_service service;
 535         boost::thread_group pool;
 536
 537         auto work = make_shared<boost::asio::io_service::work>(service);
 538
 539         int const threads = max (1, Config::instance()->master_encoding_threads());
 540
 541         for (int i = 0; i < threads; ++i) {
 542                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 543         }
 544
 545         std::function<void (float)> set_progress;
 546         if (job) {
 547                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 548         } else {
 549                 set_progress = [](float) {
 550                         boost::this_thread::interruption_point();
 551                 };
 552         }
 553
 554         for (auto& i: _reels) {
 555                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 556         }
 557         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 558
 559         work.reset ();
 560
 561         try {
 562                 pool.join_all ();
 563         } catch (boost::thread_interrupted) {
 564                 /* join_all was interrupted, so we need to interrupt the threads
 565                  * in our pool then try again to join them.
 566                  */
 567                 pool.interrupt_all ();
 568                 pool.join_all ();
 569         }
 570
 571         service.stop ();
 572 }
 573
 574
 575 /** @param output_dcp Path to DCP folder to write */
 576 void
 577 Writer::finish (boost::filesystem::path output_dcp)
 578 {
 579         if (_thread.joinable()) {
 580                 LOG_GENERAL_NC ("Terminating writer thread");
 581                 terminate_thread (true);
 582         }
 583
 584         LOG_GENERAL_NC ("Finishing ReelWriters");
 585
 586         for (auto& i: _reels) {
 587                 write_hanging_text (i);
 588                 i.finish (output_dcp);
 589         }
 590
 591         LOG_GENERAL_NC ("Writing XML");
 592
 593         dcp::DCP dcp (output_dcp);
 594
 595         auto cpl = make_shared<dcp::CPL>(
 596                 film()->dcp_name(),
 597                 film()->dcp_content_type()->libdcp_kind(),
 598                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 599                 );
 600
 601         dcp.add (cpl);
 602
 603         calculate_digests ();
 604
 605         /* Add reels */
 606
 607         for (auto& i: _reels) {
 608                 cpl->add (i.create_reel(_reel_assets, _fonts, _chosen_interop_font, output_dcp, _have_subtitles, _have_closed_captions));
 609         }
 610
 611         /* Add metadata */
 612
 613         auto creator = Config::instance()->dcp_creator();
 614         if (creator.empty()) {
 615                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 616         }
 617
 618         auto issuer = Config::instance()->dcp_issuer();
 619         if (issuer.empty()) {
 620                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 621         }
 622
 623         cpl->set_creator (creator);
 624         cpl->set_issuer (issuer);
 625
 626         cpl->set_ratings (film()->ratings());
 627
 628         vector<dcp::ContentVersion> cv;
 629         for (auto i: film()->content_versions()) {
 630                 cv.push_back (dcp::ContentVersion(i));
 631         }
 632         if (cv.empty()) {
 633                 cv = { dcp::ContentVersion("1") };
 634         }
 635         cpl->set_content_versions (cv);
 636
 637         cpl->set_full_content_title_text (film()->name());
 638         cpl->set_full_content_title_text_language (film()->name_language());
 639         if (film()->release_territory()) {
 640                 cpl->set_release_territory (*film()->release_territory());
 641         }
 642         cpl->set_version_number (film()->version_number());
 643         cpl->set_status (film()->status());
 644         if (film()->chain()) {
 645                 cpl->set_chain (*film()->chain());
 646         }
 647         if (film()->distributor()) {
 648                 cpl->set_distributor (*film()->distributor());
 649         }
 650         if (film()->facility()) {
 651                 cpl->set_facility (*film()->facility());
 652         }
 653         if (film()->luminance()) {
 654                 cpl->set_luminance (*film()->luminance());
 655         }
 656         if (film()->sign_language_video_language()) {
 657                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 658         }
 659
 660         dcp::MCASoundField field;
 661         if (film()->audio_channels() == 2) {
 662                 field = dcp::MCASoundField::STEREO;
 663         } else if (film()->audio_channels() <= 6) {
 664                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 665         } else {
 666                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 667         }
 668
 669         dcp::MainSoundConfiguration msc (field, film()->audio_channels());
 670         for (auto i: film()->mapped_audio_channels()) {
 671                 if (static_cast<int>(i) < film()->audio_channels()) {
 672                         msc.set_mapping (i, static_cast<dcp::Channel>(i));
 673                 }
 674         }
 675
 676         cpl->set_main_sound_configuration (msc.to_string());
 677         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 678         cpl->set_main_picture_stored_area (film()->frame_size());
 679
 680         auto active_area = film()->active_area();
 681         if (active_area.width > 0 && active_area.height > 0) {
 682                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 683                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 684         }
 685
 686         auto sl = film()->subtitle_languages().second;
 687         if (!sl.empty()) {
 688                 cpl->set_additional_subtitle_languages(sl);
 689         }
 690
 691         auto signer = Config::instance()->signer_chain();
 692         /* We did check earlier, but check again here to be on the safe side */
 693         string reason;
 694         if (!signer->valid (&reason)) {
 695                 throw InvalidSignerError (reason);
 696         }
 697
 698         dcp.set_issuer(issuer);
 699         dcp.set_creator(creator);
 700         dcp.set_annotation_text(film()->dcp_name());
 701
 702         dcp.write_xml (signer, Config::instance()->dcp_metadata_filename_format());
 703
 704         LOG_GENERAL (
 705                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 706                 );
 707
 708         write_cover_sheet (output_dcp);
 709 }
 710
 711
 712 void
 713 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 714 {
 715         auto const cover = film()->file("COVER_SHEET.txt");
 716         dcp::File f(cover, "w");
 717         if (!f) {
 718                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 719         }
 720
 721         auto text = Config::instance()->cover_sheet ();
 722         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 723         auto cpls = film()->cpls();
 724         if (!cpls.empty()) {
 725                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 726         }
 727         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 728         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 729
 730         auto audio_language = film()->audio_language();
 731         if (audio_language) {
 732                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 733         } else {
 734                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 735         }
 736
 737         auto subtitle_languages = film()->subtitle_languages();
 738         if (subtitle_languages.first) {
 739                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 740         } else {
 741                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 742         }
 743
 744         boost::uintmax_t size = 0;
 745         for (
 746                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 747                 i != boost::filesystem::recursive_directory_iterator();
 748                 ++i) {
 749                 if (boost::filesystem::is_regular_file (i->path())) {
 750                         size += boost::filesystem::file_size (i->path());
 751                 }
 752         }
 753
 754         if (size > (1000000000L)) {
 755                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 756         } else {
 757                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 758         }
 759
 760         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 761         auto description = String::compose("%1.%2", ch.first, ch.second);
 762
 763         if (description == "0.0") {
 764                 description = _("None");
 765         } else if (description == "1.0") {
 766                 description = _("Mono");
 767         } else if (description == "2.0") {
 768                 description = _("Stereo");
 769         }
 770         boost::algorithm::replace_all (text, "$AUDIO", description);
 771
 772         auto const hmsf = film()->length().split(film()->video_frame_rate());
 773         string length;
 774         if (hmsf.h == 0 && hmsf.m == 0) {
 775                 length = String::compose("%1s", hmsf.s);
 776         } else if (hmsf.h == 0 && hmsf.m > 0) {
 777                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 778         } else if (hmsf.h > 0 && hmsf.m > 0) {
 779                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 780         }
 781
 782         boost::algorithm::replace_all (text, "$LENGTH", length);
 783
 784         f.checked_write(text.c_str(), text.length());
 785 }
 786
 787
 788 /** @param frame Frame index within the whole DCP.
 789  *  @return true if we can fake-write this frame.
 790  */
 791 bool
 792 Writer::can_fake_write (Frame frame) const
 793 {
 794         if (film()->encrypted()) {
 795                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 796                 return false;
 797         }
 798
 799         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 800            parameters in the asset writer.
 801         */
 802
 803         auto const & reel = _reels[video_reel(frame)];
 804
 805         /* Make frame relative to the start of the reel */
 806         frame -= reel.start ();
 807         return (frame != 0 && frame < reel.first_nonexistent_frame());
 808 }
 809
 810
 811 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 812 void
 813 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 814 {
 815         vector<ReelWriter>::iterator* reel = nullptr;
 816
 817         switch (type) {
 818         case TextType::OPEN_SUBTITLE:
 819                 reel = &_subtitle_reel;
 820                 _have_subtitles = true;
 821                 break;
 822         case TextType::CLOSED_CAPTION:
 823                 DCPOMATIC_ASSERT (track);
 824                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 825                 reel = &_caption_reels[*track];
 826                 _have_closed_captions.insert (*track);
 827                 break;
 828         default:
 829                 DCPOMATIC_ASSERT (false);
 830         }
 831
 832         DCPOMATIC_ASSERT (*reel != _reels.end());
 833         while ((*reel)->period().to <= period.from) {
 834                 ++(*reel);
 835                 DCPOMATIC_ASSERT (*reel != _reels.end());
 836                 write_hanging_text (**reel);
 837         }
 838
 839         auto back_off = [this](DCPTimePeriod period) {
 840                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 841                 return period;
 842         };
 843
 844         if (period.to > (*reel)->period().to) {
 845                 /* This text goes off the end of the reel.  Store parts of it that should go into
 846                  * other reels.
 847                  */
 848                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 849                         auto overlap = i->period().overlap(period);
 850                         if (overlap) {
 851                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 852                         }
 853                 }
 854                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 855                  * for subtitles being too close together.
 856                  */
 857                 period.to = (*reel)->period().to;
 858                 period = back_off(period);
 859         }
 860
 861         (*reel)->write(text, type, track, period, _fonts);
 862 }
 863
 864
 865 void
 866 Writer::write (vector<shared_ptr<Font>> fonts)
 867 {
 868         if (fonts.empty()) {
 869                 return;
 870         }
 871
 872         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 873         auto fix_id = [](string id) {
 874                 return id.empty() ? "font" : id;
 875         };
 876
 877         if (film()->interop()) {
 878                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 879                  * even write them as they upset some validators.  Set up _fonts so that every
 880                  * font used by any subtitle will be written with the same ID.
 881                  */
 882                 for (size_t i = 0; i < fonts.size(); ++i) {
 883                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 884                 }
 885                 _chosen_interop_font = fonts[0];
 886         } else {
 887                 set<string> used_ids;
 888
 889                 /* Return the index of a _N at the end of a string, or string::npos */
 890                 auto underscore_number_position = [](string s) {
 891                         auto last_underscore = s.find_last_of("_");
 892                         if (last_underscore == string::npos) {
 893                                 return string::npos;
 894                         }
 895
 896                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 897                                 if (!isdigit(s[i])) {
 898                                         return string::npos;
 899                                 }
 900                         }
 901
 902                         return last_underscore;
 903                 };
 904
 905                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 906                 for (auto font: fonts) {
 907                         auto id = fix_id(font->id());
 908                         if (used_ids.find(id) == used_ids.end()) {
 909                                 /* This ID is unique so we can just use it as-is */
 910                                 _fonts.put(font, id);
 911                                 used_ids.insert(id);
 912                         } else {
 913                                 auto end = underscore_number_position(id);
 914                                 if (end == string::npos) {
 915                                         /* This string has no _N suffix, so add one */
 916                                         id += "_0";
 917                                         end = underscore_number_position(id);
 918                                 }
 919
 920                                 ++end;
 921
 922                                 /* Increment the suffix until we find a unique one */
 923                                 auto number = dcp::raw_convert<int>(id.substr(end));
 924                                 while (used_ids.find(id) != used_ids.end()) {
 925                                         ++number;
 926                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 927                                 }
 928                                 used_ids.insert(id);
 929                         }
 930                         _fonts.put(font, id);
 931                 }
 932
 933                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 934         }
 935 }
 936
 937
 938 bool
 939 operator< (QueueItem const & a, QueueItem const & b)
 940 {
 941         if (a.reel != b.reel) {
 942                 return a.reel < b.reel;
 943         }
 944
 945         if (a.frame != b.frame) {
 946                 return a.frame < b.frame;
 947         }
 948
 949         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 950 }
 951
 952
 953 bool
 954 operator== (QueueItem const & a, QueueItem const & b)
 955 {
 956         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 957 }
 958
 959
 960 void
 961 Writer::set_encoder_threads (int threads)
 962 {
 963         boost::mutex::scoped_lock lm (_state_mutex);
 964         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 965         _maximum_queue_size = threads * 16;
 966 }
 967
 968
 969 void
 970 Writer::write (ReferencedReelAsset asset)
 971 {
 972         _reel_assets.push_back (asset);
 973 }
 974
 975
 976 size_t
 977 Writer::video_reel (int frame) const
 978 {
 979         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 980         size_t i = 0;
 981         while (i < _reels.size() && !_reels[i].period().contains (t)) {
 982                 ++i;
 983         }
 984
 985         DCPOMATIC_ASSERT (i < _reels.size ());
 986         return i;
 987 }
 988
 989
 990 void
 991 Writer::set_digest_progress (Job* job, float progress)
 992 {
 993         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 994
 995         _digest_progresses[boost::this_thread::get_id()] = progress;
 996         float min_progress = FLT_MAX;
 997         for (auto const& i: _digest_progresses) {
 998                 min_progress = min (min_progress, i.second);
 999         }
1000
1001         job->set_progress (min_progress);
1002
1003         Waker waker;
1004         waker.nudge ();
1005
1006         boost::this_thread::interruption_point();
1007 }
1008
1009
1010 /** Calculate hashes for any referenced MXF assets which do not already have one */
1011 void
1012 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
1013 try
1014 {
1015         for (auto const& i: _reel_assets) {
1016                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1017                 if (file && !file->hash()) {
1018                         file->asset_ref().asset()->hash (set_progress);
1019                         file->set_hash (file->asset_ref().asset()->hash());
1020                 }
1021         }
1022 } catch (boost::thread_interrupted) {
1023         /* set_progress contains an interruption_point, so any of these methods
1024          * may throw thread_interrupted, at which point we just give up.
1025          */
1026 }
1027
1028
1029 void
1030 Writer::write_hanging_text (ReelWriter& reel)
1031 {
1032         vector<HangingText> new_hanging_texts;
1033         for (auto i: _hanging_texts) {
1034                 if (i.period.from == reel.period().from) {
1035                         reel.write (i.text, i.type, i.track, i.period, _fonts);
1036                 } else {
1037                         new_hanging_texts.push_back (i);
1038                 }
1039         }
1040         _hanging_texts = new_hanging_texts;
1041 }