src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "cross.h"
  27 #include "dcp_content_type.h"
  28 #include "dcp_video.h"
  29 #include "dcpomatic_log.h"
  30 #include "film.h"
  31 #include "job.h"
  32 #include "log.h"
  33 #include "ratio.h"
  34 #include "reel_writer.h"
  35 #include "text_content.h"
  36 #include "util.h"
  37 #include "version.h"
  38 #include "writer.h"
  39 #include <dcp/cpl.h>
  40 #include <dcp/locale_convert.h>
  41 #include <dcp/raw_convert.h>
  42 #include <dcp/reel_file_asset.h>
  43 #include <cerrno>
  44 #include <cfloat>
  45 #include <set>
  46
  47 #include "i18n.h"
  48
  49
  50 /* OS X strikes again */
  51 #undef set_key
  52
  53
  54 using std::cout;
  55 using std::dynamic_pointer_cast;
  56 using std::make_shared;
  57 using std::max;
  58 using std::min;
  59 using std::shared_ptr;
  60 using std::set;
  61 using std::string;
  62 using std::vector;
  63 using std::weak_ptr;
  64 using boost::optional;
  65 #if BOOST_VERSION >= 106100
  66 using namespace boost::placeholders;
  67 #endif
  68 using dcp::Data;
  69 using dcp::ArrayData;
  70 using namespace dcpomatic;
  71
  72
  73 /** @param j Job to report progress to, or 0.
  74  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  75  */
  76 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
  77         : WeakConstFilm (weak_film)
  78         , _job (j)
  79         /* These will be reset to sensible values when J2KEncoder is created */
  80         , _maximum_frames_in_memory (8)
  81         , _maximum_queue_size (8)
  82         , _text_only (text_only)
  83 {
  84         auto job = _job.lock ();
  85
  86         int reel_index = 0;
  87         auto const reels = film()->reels();
  88         for (auto p: reels) {
  89                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  90         }
  91
  92         _last_written.resize (reels.size());
  93
  94         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  95            and captions arrive to the Writer in sequence.  This is not so for video.
  96         */
  97         _audio_reel = _reels.begin ();
  98         _subtitle_reel = _reels.begin ();
  99         for (auto i: film()->closed_caption_tracks()) {
 100                 _caption_reels[i] = _reels.begin ();
 101         }
 102         _atmos_reel = _reels.begin ();
 103
 104         /* Check that the signer is OK */
 105         string reason;
 106         if (!Config::instance()->signer_chain()->valid(&reason)) {
 107                 throw InvalidSignerError (reason);
 108         }
 109 }
 110
 111
 112 void
 113 Writer::start ()
 114 {
 115         if (!_text_only) {
 116                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 117 #ifdef DCPOMATIC_LINUX
 118                 pthread_setname_np (_thread.native_handle(), "writer");
 119 #endif
 120         }
 121 }
 122
 123
 124 Writer::~Writer ()
 125 {
 126         if (!_text_only) {
 127                 terminate_thread (false);
 128         }
 129 }
 130
 131
 132 /** Pass a video frame to the writer for writing to disk at some point.
 133  *  This method can be called with frames out of order.
 134  *  @param encoded JPEG2000-encoded data.
 135  *  @param frame Frame index within the DCP.
 136  *  @param eyes Eyes that this frame image is for.
 137  */
 138 void
 139 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 140 {
 141         boost::mutex::scoped_lock lock (_state_mutex);
 142
 143         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 144                 /* There are too many full frames in memory; wake the main writer thread and
 145                    wait until it sorts everything out */
 146                 _empty_condition.notify_all ();
 147                 _full_condition.wait (lock);
 148         }
 149
 150         QueueItem qi;
 151         qi.type = QueueItem::Type::FULL;
 152         qi.encoded = encoded;
 153         qi.reel = video_reel (frame);
 154         qi.frame = frame - _reels[qi.reel].start ();
 155
 156         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 157
 158         qi.eyes = eyes;
 159         _queue.push_back(qi);
 160         ++_queued_full_in_memory;
 161
 162         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 163         _empty_condition.notify_all ();
 164 }
 165
 166
 167 bool
 168 Writer::can_repeat (Frame frame) const
 169 {
 170         return frame > _reels[video_reel(frame)].start();
 171 }
 172
 173
 174 /** Repeat the last frame that was written to a reel as a new frame.
 175  *  @param frame Frame index within the DCP of the new (repeated) frame.
 176  *  @param eyes Eyes that this repeated frame image is for.
 177  */
 178 void
 179 Writer::repeat (Frame frame, Eyes eyes)
 180 {
 181         boost::mutex::scoped_lock lock (_state_mutex);
 182
 183         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 184                 /* The queue is too big, and the main writer thread can run and fix it, so
 185                    wake it and wait until it has done.
 186                 */
 187                 _empty_condition.notify_all ();
 188                 _full_condition.wait (lock);
 189         }
 190
 191         QueueItem qi;
 192         qi.type = QueueItem::Type::REPEAT;
 193         qi.reel = video_reel (frame);
 194         qi.frame = frame - _reels[qi.reel].start ();
 195         if (film()->three_d() && eyes == Eyes::BOTH) {
 196                 qi.eyes = Eyes::LEFT;
 197                 _queue.push_back (qi);
 198                 qi.eyes = Eyes::RIGHT;
 199                 _queue.push_back (qi);
 200         } else {
 201                 qi.eyes = eyes;
 202                 _queue.push_back (qi);
 203         }
 204
 205         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 206         _empty_condition.notify_all ();
 207 }
 208
 209
 210 void
 211 Writer::fake_write (Frame frame, Eyes eyes)
 212 {
 213         boost::mutex::scoped_lock lock (_state_mutex);
 214
 215         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 216                 /* The queue is too big, and the main writer thread can run and fix it, so
 217                    wake it and wait until it has done.
 218                 */
 219                 _empty_condition.notify_all ();
 220                 _full_condition.wait (lock);
 221         }
 222
 223         size_t const reel = video_reel (frame);
 224         Frame const frame_in_reel = frame - _reels[reel].start ();
 225
 226         QueueItem qi;
 227         qi.type = QueueItem::Type::FAKE;
 228
 229         {
 230                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 231                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 232         }
 233
 234         qi.reel = reel;
 235         qi.frame = frame_in_reel;
 236         if (film()->three_d() && eyes == Eyes::BOTH) {
 237                 qi.eyes = Eyes::LEFT;
 238                 _queue.push_back (qi);
 239                 qi.eyes = Eyes::RIGHT;
 240                 _queue.push_back (qi);
 241         } else {
 242                 qi.eyes = eyes;
 243                 _queue.push_back (qi);
 244         }
 245
 246         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 247         _empty_condition.notify_all ();
 248 }
 249
 250
 251 /** Write some audio frames to the DCP.
 252  *  @param audio Audio data.
 253  *  @param time Time of this data within the DCP.
 254  *  This method is not thread safe.
 255  */
 256 void
 257 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 258 {
 259         DCPOMATIC_ASSERT (audio);
 260
 261         int const afr = film()->audio_frame_rate();
 262
 263         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 264
 265         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 266
 267         DCPTime t = time;
 268         while (t < end) {
 269
 270                 if (_audio_reel == _reels.end ()) {
 271                         /* This audio is off the end of the last reel; ignore it */
 272                         return;
 273                 }
 274
 275                 if (end <= _audio_reel->period().to) {
 276                         /* Easy case: we can write all the audio to this reel */
 277                         _audio_reel->write (audio);
 278                         t = end;
 279                 } else if (_audio_reel->period().to <= t) {
 280                         /* This reel is entirely before the start of our audio; just skip the reel */
 281                         ++_audio_reel;
 282                 } else {
 283                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 284                         DCPTime part_lengths[2] = {
 285                                 _audio_reel->period().to - t,
 286                                 end - _audio_reel->period().to
 287                         };
 288
 289                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 290                         Frame part_frames[2] = {
 291                                 part_lengths[0].frames_ceil(afr),
 292                                 part_lengths[1].frames_floor(afr)
 293                         };
 294
 295                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 296
 297                         if (part_frames[0]) {
 298                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 299                                 _audio_reel->write (part);
 300                         }
 301
 302                         if (part_frames[1]) {
 303                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 304                         } else {
 305                                 audio.reset ();
 306                         }
 307
 308                         ++_audio_reel;
 309                         t += part_lengths[0];
 310                 }
 311         }
 312 }
 313
 314
 315 void
 316 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 317 {
 318         if (_atmos_reel->period().to == time) {
 319                 ++_atmos_reel;
 320                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 321         }
 322
 323         /* We assume that we get a video frame's worth of data here */
 324         _atmos_reel->write (atmos, metadata);
 325 }
 326
 327
 328 /** Caller must hold a lock on _state_mutex */
 329 bool
 330 Writer::have_sequenced_image_at_queue_head ()
 331 {
 332         if (_queue.empty ()) {
 333                 return false;
 334         }
 335
 336         _queue.sort ();
 337         auto const & f = _queue.front();
 338         return _last_written[f.reel].next(f);
 339 }
 340
 341
 342 bool
 343 Writer::LastWritten::next (QueueItem qi) const
 344 {
 345         if (qi.eyes == Eyes::BOTH) {
 346                 /* 2D */
 347                 return qi.frame == (_frame + 1);
 348         }
 349
 350         /* 3D */
 351
 352         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 353                 return true;
 354         }
 355
 356         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 357                 return true;
 358         }
 359
 360         return false;
 361 }
 362
 363
 364 void
 365 Writer::LastWritten::update (QueueItem qi)
 366 {
 367         _frame = qi.frame;
 368         _eyes = qi.eyes;
 369 }
 370
 371
 372 void
 373 Writer::thread ()
 374 try
 375 {
 376         start_of_thread ("Writer");
 377
 378         while (true)
 379         {
 380                 boost::mutex::scoped_lock lock (_state_mutex);
 381
 382                 while (true) {
 383
 384                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 385                                 /* We've got something to do: go and do it */
 386                                 break;
 387                         }
 388
 389                         /* Nothing to do: wait until something happens which may indicate that we do */
 390                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 391                         _empty_condition.wait (lock);
 392                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 393                 }
 394
 395                 /* We stop here if we have been asked to finish, and if either the queue
 396                    is empty or we do not have a sequenced image at its head (if this is the
 397                    case we will never terminate as no new frames will be sent once
 398                    _finish is true).
 399                 */
 400                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 401                         /* (Hopefully temporarily) log anything that was not written */
 402                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 403                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 404                                 for (auto const& i: _queue) {
 405                                         if (i.type == QueueItem::Type::FULL) {
 406                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 407                                         } else {
 408                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 409                                         }
 410                                 }
 411                         }
 412                         return;
 413                 }
 414
 415                 /* Write any frames that we can write; i.e. those that are in sequence. */
 416                 while (have_sequenced_image_at_queue_head ()) {
 417                         auto qi = _queue.front ();
 418                         _last_written[qi.reel].update (qi);
 419                         _queue.pop_front ();
 420                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 421                                 --_queued_full_in_memory;
 422                         }
 423
 424                         lock.unlock ();
 425
 426                         auto& reel = _reels[qi.reel];
 427
 428                         switch (qi.type) {
 429                         case QueueItem::Type::FULL:
 430                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 431                                 if (!qi.encoded) {
 432                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 433                                 }
 434                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 435                                 ++_full_written;
 436                                 break;
 437                         case QueueItem::Type::FAKE:
 438                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 439                                 reel.fake_write (qi.size);
 440                                 ++_fake_written;
 441                                 break;
 442                         case QueueItem::Type::REPEAT:
 443                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 444                                 reel.repeat_write (qi.frame, qi.eyes);
 445                                 ++_repeat_written;
 446                                 break;
 447                         }
 448
 449                         lock.lock ();
 450                         _full_condition.notify_all ();
 451                 }
 452
 453                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 454                         /* Too many frames in memory which can't yet be written to the stream.
 455                            Write some FULL frames to disk.
 456                         */
 457
 458                         /* Find one from the back of the queue */
 459                         _queue.sort ();
 460                         auto i = _queue.rbegin ();
 461                         while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
 462                                 ++i;
 463                         }
 464
 465                         DCPOMATIC_ASSERT (i != _queue.rend());
 466                         ++_pushed_to_disk;
 467                         /* For the log message below */
 468                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 469                         lock.unlock ();
 470
 471                         /* i is valid here, even though we don't hold a lock on the mutex,
 472                            since list iterators are unaffected by insertion and only this
 473                            thread could erase the last item in the list.
 474                         */
 475
 476                         LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
 477
 478                         i->encoded->write_via_temp (
 479                                 film()->j2c_path(i->reel, i->frame, i->eyes, true),
 480                                 film()->j2c_path(i->reel, i->frame, i->eyes, false)
 481                                 );
 482
 483                         lock.lock ();
 484                         i->encoded.reset ();
 485                         --_queued_full_in_memory;
 486                         _full_condition.notify_all ();
 487                 }
 488         }
 489 }
 490 catch (...)
 491 {
 492         store_current ();
 493 }
 494
 495
 496 void
 497 Writer::terminate_thread (bool can_throw)
 498 {
 499         boost::this_thread::disable_interruption dis;
 500
 501         boost::mutex::scoped_lock lock (_state_mutex);
 502
 503         _finish = true;
 504         _empty_condition.notify_all ();
 505         _full_condition.notify_all ();
 506         lock.unlock ();
 507
 508         try {
 509                 _thread.join ();
 510         } catch (...) {}
 511
 512         if (can_throw) {
 513                 rethrow ();
 514         }
 515 }
 516
 517
 518 void
 519 Writer::calculate_digests ()
 520 {
 521         auto job = _job.lock ();
 522         if (job) {
 523                 job->sub (_("Computing digests"));
 524         }
 525
 526         boost::asio::io_service service;
 527         boost::thread_group pool;
 528
 529         auto work = make_shared<boost::asio::io_service::work>(service);
 530
 531         int const threads = max (1, Config::instance()->master_encoding_threads());
 532
 533         for (int i = 0; i < threads; ++i) {
 534                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 535         }
 536
 537         std::function<void (float)> set_progress;
 538         if (job) {
 539                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 540         } else {
 541                 set_progress = [](float) {
 542                         boost::this_thread::interruption_point();
 543                 };
 544         }
 545
 546         for (auto& i: _reels) {
 547                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 548         }
 549         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 550
 551         work.reset ();
 552
 553         try {
 554                 pool.join_all ();
 555         } catch (boost::thread_interrupted) {
 556                 /* join_all was interrupted, so we need to interrupt the threads
 557                  * in our pool then try again to join them.
 558                  */
 559                 pool.interrupt_all ();
 560                 pool.join_all ();
 561         }
 562
 563         service.stop ();
 564 }
 565
 566
 567 /** @param output_dcp Path to DCP folder to write */
 568 void
 569 Writer::finish (boost::filesystem::path output_dcp)
 570 {
 571         if (_thread.joinable()) {
 572                 LOG_GENERAL_NC ("Terminating writer thread");
 573                 terminate_thread (true);
 574         }
 575
 576         LOG_GENERAL_NC ("Finishing ReelWriters");
 577
 578         for (auto& i: _reels) {
 579                 write_hanging_text (i);
 580                 i.finish (output_dcp);
 581         }
 582
 583         LOG_GENERAL_NC ("Writing XML");
 584
 585         dcp::DCP dcp (output_dcp);
 586
 587         auto cpl = make_shared<dcp::CPL>(
 588                 film()->dcp_name(),
 589                 film()->dcp_content_type()->libdcp_kind(),
 590                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 591                 );
 592
 593         dcp.add (cpl);
 594
 595         calculate_digests ();
 596
 597         /* Add reels */
 598
 599         for (auto& i: _reels) {
 600                 cpl->add (i.create_reel(_reel_assets, _fonts, _chosen_interop_font, output_dcp, _have_subtitles, _have_closed_captions));
 601         }
 602
 603         /* Add metadata */
 604
 605         auto creator = Config::instance()->dcp_creator();
 606         if (creator.empty()) {
 607                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 608         }
 609
 610         auto issuer = Config::instance()->dcp_issuer();
 611         if (issuer.empty()) {
 612                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 613         }
 614
 615         cpl->set_creator (creator);
 616         cpl->set_issuer (issuer);
 617
 618         cpl->set_ratings (film()->ratings());
 619
 620         vector<dcp::ContentVersion> cv;
 621         for (auto i: film()->content_versions()) {
 622                 cv.push_back (dcp::ContentVersion(i));
 623         }
 624         if (cv.empty()) {
 625                 cv = { dcp::ContentVersion("1") };
 626         }
 627         cpl->set_content_versions (cv);
 628
 629         cpl->set_full_content_title_text (film()->name());
 630         cpl->set_full_content_title_text_language (film()->name_language());
 631         if (film()->release_territory()) {
 632                 cpl->set_release_territory (*film()->release_territory());
 633         }
 634         cpl->set_version_number (film()->version_number());
 635         cpl->set_status (film()->status());
 636         if (film()->chain()) {
 637                 cpl->set_chain (*film()->chain());
 638         }
 639         if (film()->distributor()) {
 640                 cpl->set_distributor (*film()->distributor());
 641         }
 642         if (film()->facility()) {
 643                 cpl->set_facility (*film()->facility());
 644         }
 645         if (film()->luminance()) {
 646                 cpl->set_luminance (*film()->luminance());
 647         }
 648         if (film()->sign_language_video_language()) {
 649                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 650         }
 651
 652         dcp::MCASoundField field;
 653         if (film()->audio_channels() == 2) {
 654                 field = dcp::MCASoundField::STEREO;
 655         } else if (film()->audio_channels() <= 6) {
 656                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 657         } else {
 658                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 659         }
 660
 661         dcp::MainSoundConfiguration msc (field, film()->audio_channels());
 662         for (auto i: film()->mapped_audio_channels()) {
 663                 if (static_cast<int>(i) < film()->audio_channels()) {
 664                         msc.set_mapping (i, static_cast<dcp::Channel>(i));
 665                 }
 666         }
 667
 668         cpl->set_main_sound_configuration (msc.to_string());
 669         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 670         cpl->set_main_picture_stored_area (film()->frame_size());
 671
 672         auto active_area = film()->active_area();
 673         if (active_area.width > 0 && active_area.height > 0) {
 674                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 675                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 676         }
 677
 678         auto sl = film()->subtitle_languages().second;
 679         if (!sl.empty()) {
 680                 cpl->set_additional_subtitle_languages(sl);
 681         }
 682
 683         auto signer = Config::instance()->signer_chain();
 684         /* We did check earlier, but check again here to be on the safe side */
 685         string reason;
 686         if (!signer->valid (&reason)) {
 687                 throw InvalidSignerError (reason);
 688         }
 689
 690         dcp.set_issuer(issuer);
 691         dcp.set_creator(creator);
 692         dcp.set_annotation_text(film()->dcp_name());
 693
 694         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 695
 696         LOG_GENERAL (
 697                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 698                 );
 699
 700         write_cover_sheet (output_dcp);
 701 }
 702
 703
 704 void
 705 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 706 {
 707         auto const cover = film()->file("COVER_SHEET.txt");
 708         dcp::File f(cover, "w");
 709         if (!f) {
 710                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 711         }
 712
 713         auto text = Config::instance()->cover_sheet ();
 714         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 715         auto cpls = film()->cpls();
 716         if (!cpls.empty()) {
 717                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 718         }
 719         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 720         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 721
 722         auto audio_language = film()->audio_language();
 723         if (audio_language) {
 724                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 725         } else {
 726                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 727         }
 728
 729         auto subtitle_languages = film()->subtitle_languages();
 730         if (subtitle_languages.first) {
 731                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 732         } else {
 733                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 734         }
 735
 736         boost::uintmax_t size = 0;
 737         for (
 738                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 739                 i != boost::filesystem::recursive_directory_iterator();
 740                 ++i) {
 741                 if (boost::filesystem::is_regular_file (i->path())) {
 742                         size += boost::filesystem::file_size (i->path());
 743                 }
 744         }
 745
 746         if (size > (1000000000L)) {
 747                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 748         } else {
 749                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 750         }
 751
 752         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 753         auto description = String::compose("%1.%2", ch.first, ch.second);
 754
 755         if (description == "0.0") {
 756                 description = _("None");
 757         } else if (description == "1.0") {
 758                 description = _("Mono");
 759         } else if (description == "2.0") {
 760                 description = _("Stereo");
 761         }
 762         boost::algorithm::replace_all (text, "$AUDIO", description);
 763
 764         auto const hmsf = film()->length().split(film()->video_frame_rate());
 765         string length;
 766         if (hmsf.h == 0 && hmsf.m == 0) {
 767                 length = String::compose("%1s", hmsf.s);
 768         } else if (hmsf.h == 0 && hmsf.m > 0) {
 769                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 770         } else if (hmsf.h > 0 && hmsf.m > 0) {
 771                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 772         }
 773
 774         boost::algorithm::replace_all (text, "$LENGTH", length);
 775
 776         f.checked_write(text.c_str(), text.length());
 777 }
 778
 779
 780 /** @param frame Frame index within the whole DCP.
 781  *  @return true if we can fake-write this frame.
 782  */
 783 bool
 784 Writer::can_fake_write (Frame frame) const
 785 {
 786         if (film()->encrypted()) {
 787                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 788                 return false;
 789         }
 790
 791         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 792            parameters in the asset writer.
 793         */
 794
 795         auto const & reel = _reels[video_reel(frame)];
 796
 797         /* Make frame relative to the start of the reel */
 798         frame -= reel.start ();
 799         return (frame != 0 && frame < reel.first_nonexistent_frame());
 800 }
 801
 802
 803 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 804 void
 805 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 806 {
 807         vector<ReelWriter>::iterator* reel = nullptr;
 808
 809         switch (type) {
 810         case TextType::OPEN_SUBTITLE:
 811                 reel = &_subtitle_reel;
 812                 _have_subtitles = true;
 813                 break;
 814         case TextType::CLOSED_CAPTION:
 815                 DCPOMATIC_ASSERT (track);
 816                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 817                 reel = &_caption_reels[*track];
 818                 _have_closed_captions.insert (*track);
 819                 break;
 820         default:
 821                 DCPOMATIC_ASSERT (false);
 822         }
 823
 824         DCPOMATIC_ASSERT (*reel != _reels.end());
 825         while ((*reel)->period().to <= period.from) {
 826                 ++(*reel);
 827                 DCPOMATIC_ASSERT (*reel != _reels.end());
 828                 write_hanging_text (**reel);
 829         }
 830
 831         auto back_off = [this](DCPTimePeriod period) {
 832                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 833                 return period;
 834         };
 835
 836         if (period.to > (*reel)->period().to) {
 837                 /* This text goes off the end of the reel.  Store parts of it that should go into
 838                  * other reels.
 839                  */
 840                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 841                         auto overlap = i->period().overlap(period);
 842                         if (overlap) {
 843                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 844                         }
 845                 }
 846                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 847                  * for subtitles being too close together.
 848                  */
 849                 period.to = (*reel)->period().to;
 850                 period = back_off(period);
 851         }
 852
 853         (*reel)->write(text, type, track, period, _fonts);
 854 }
 855
 856
 857 void
 858 Writer::write (vector<shared_ptr<Font>> fonts)
 859 {
 860         if (fonts.empty()) {
 861                 return;
 862         }
 863
 864         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 865         auto fix_id = [](string id) {
 866                 return id.empty() ? "font" : id;
 867         };
 868
 869         if (film()->interop()) {
 870                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 871                  * even write them as they upset some validators.  Set up _fonts so that every
 872                  * font used by any subtitle will be written with the same ID.
 873                  */
 874                 for (size_t i = 0; i < fonts.size(); ++i) {
 875                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 876                 }
 877                 _chosen_interop_font = fonts[0];
 878         } else {
 879                 set<string> used_ids;
 880
 881                 /* Return the index of a _N at the end of a string, or string::npos */
 882                 auto underscore_number_position = [](string s) {
 883                         auto last_underscore = s.find_last_of("_");
 884                         if (last_underscore == string::npos) {
 885                                 return string::npos;
 886                         }
 887
 888                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 889                                 if (!isdigit(s[i])) {
 890                                         return string::npos;
 891                                 }
 892                         }
 893
 894                         return last_underscore;
 895                 };
 896
 897                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 898                 for (auto font: fonts) {
 899                         auto id = fix_id(font->id());
 900                         if (used_ids.find(id) == used_ids.end()) {
 901                                 /* This ID is unique so we can just use it as-is */
 902                                 _fonts.put(font, id);
 903                                 used_ids.insert(id);
 904                         } else {
 905                                 auto end = underscore_number_position(id);
 906                                 if (end == string::npos) {
 907                                         /* This string has no _N suffix, so add one */
 908                                         id += "_0";
 909                                         end = underscore_number_position(id);
 910                                 }
 911
 912                                 ++end;
 913
 914                                 /* Increment the suffix until we find a unique one */
 915                                 auto number = dcp::raw_convert<int>(id.substr(end));
 916                                 while (used_ids.find(id) != used_ids.end()) {
 917                                         ++number;
 918                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 919                                 }
 920                                 used_ids.insert(id);
 921                         }
 922                         _fonts.put(font, id);
 923                 }
 924
 925                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 926         }
 927 }
 928
 929
 930 bool
 931 operator< (QueueItem const & a, QueueItem const & b)
 932 {
 933         if (a.reel != b.reel) {
 934                 return a.reel < b.reel;
 935         }
 936
 937         if (a.frame != b.frame) {
 938                 return a.frame < b.frame;
 939         }
 940
 941         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 942 }
 943
 944
 945 bool
 946 operator== (QueueItem const & a, QueueItem const & b)
 947 {
 948         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 949 }
 950
 951
 952 void
 953 Writer::set_encoder_threads (int threads)
 954 {
 955         boost::mutex::scoped_lock lm (_state_mutex);
 956         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 957         _maximum_queue_size = threads * 16;
 958 }
 959
 960
 961 void
 962 Writer::write (ReferencedReelAsset asset)
 963 {
 964         _reel_assets.push_back (asset);
 965 }
 966
 967
 968 size_t
 969 Writer::video_reel (int frame) const
 970 {
 971         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 972         size_t i = 0;
 973         while (i < _reels.size() && !_reels[i].period().contains (t)) {
 974                 ++i;
 975         }
 976
 977         DCPOMATIC_ASSERT (i < _reels.size ());
 978         return i;
 979 }
 980
 981
 982 void
 983 Writer::set_digest_progress (Job* job, float progress)
 984 {
 985         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 986
 987         _digest_progresses[boost::this_thread::get_id()] = progress;
 988         float min_progress = FLT_MAX;
 989         for (auto const& i: _digest_progresses) {
 990                 min_progress = min (min_progress, i.second);
 991         }
 992
 993         job->set_progress (min_progress);
 994
 995         Waker waker;
 996         waker.nudge ();
 997
 998         boost::this_thread::interruption_point();
 999 }
1000
1001
1002 /** Calculate hashes for any referenced MXF assets which do not already have one */
1003 void
1004 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
1005 try
1006 {
1007         for (auto const& i: _reel_assets) {
1008                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1009                 if (file && !file->hash()) {
1010                         file->asset_ref().asset()->hash (set_progress);
1011                         file->set_hash (file->asset_ref().asset()->hash());
1012                 }
1013         }
1014 } catch (boost::thread_interrupted) {
1015         /* set_progress contains an interruption_point, so any of these methods
1016          * may throw thread_interrupted, at which point we just give up.
1017          */
1018 }
1019
1020
1021 void
1022 Writer::write_hanging_text (ReelWriter& reel)
1023 {
1024         vector<HangingText> new_hanging_texts;
1025         for (auto i: _hanging_texts) {
1026                 if (i.period.from == reel.period().from) {
1027                         reel.write (i.text, i.type, i.track, i.period, _fonts);
1028                 } else {
1029                         new_hanging_texts.push_back (i);
1030                 }
1031         }
1032         _hanging_texts = new_hanging_texts;
1033 }