src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of DCP-o-matic.
   5
   6     DCP-o-matic is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     DCP-o-matic is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 */
  20
  21
  22 #include "audio_buffers.h"
  23 #include "audio_mapping.h"
  24 #include "compose.hpp"
  25 #include "config.h"
  26 #include "cross.h"
  27 #include "dcp_content_type.h"
  28 #include "dcp_video.h"
  29 #include "dcpomatic_log.h"
  30 #include "film.h"
  31 #include "job.h"
  32 #include "log.h"
  33 #include "ratio.h"
  34 #include "reel_writer.h"
  35 #include "text_content.h"
  36 #include "util.h"
  37 #include "version.h"
  38 #include "writer.h"
  39 #include <dcp/cpl.h>
  40 #include <dcp/locale_convert.h>
  41 #include <dcp/raw_convert.h>
  42 #include <dcp/reel_file_asset.h>
  43 #include <cerrno>
  44 #include <cfloat>
  45 #include <set>
  46
  47 #include "i18n.h"
  48
  49
  50 /* OS X strikes again */
  51 #undef set_key
  52
  53
  54 using std::cout;
  55 using std::dynamic_pointer_cast;
  56 using std::make_shared;
  57 using std::max;
  58 using std::min;
  59 using std::shared_ptr;
  60 using std::set;
  61 using std::string;
  62 using std::vector;
  63 using std::weak_ptr;
  64 using boost::optional;
  65 #if BOOST_VERSION >= 106100
  66 using namespace boost::placeholders;
  67 #endif
  68 using dcp::Data;
  69 using dcp::ArrayData;
  70 using namespace dcpomatic;
  71
  72
  73 /** @param j Job to report progress to, or 0.
  74  *  @param text_only true to enable only the text (subtitle/ccap) parts of the writer.
  75  */
  76 Writer::Writer (weak_ptr<const Film> weak_film, weak_ptr<Job> j, bool text_only)
  77         : WeakConstFilm (weak_film)
  78         , _job (j)
  79         /* These will be reset to sensible values when J2KEncoder is created */
  80         , _maximum_frames_in_memory (8)
  81         , _maximum_queue_size (8)
  82         , _text_only (text_only)
  83 {
  84         auto job = _job.lock ();
  85
  86         int reel_index = 0;
  87         auto const reels = film()->reels();
  88         for (auto p: reels) {
  89                 _reels.push_back (ReelWriter(weak_film, p, job, reel_index++, reels.size(), text_only));
  90         }
  91
  92         _last_written.resize (reels.size());
  93
  94         /* We can keep track of the current audio, subtitle and closed caption reels easily because audio
  95            and captions arrive to the Writer in sequence.  This is not so for video.
  96         */
  97         _audio_reel = _reels.begin ();
  98         _subtitle_reel = _reels.begin ();
  99         for (auto i: film()->closed_caption_tracks()) {
 100                 _caption_reels[i] = _reels.begin ();
 101         }
 102         _atmos_reel = _reels.begin ();
 103
 104         /* Check that the signer is OK */
 105         string reason;
 106         if (!Config::instance()->signer_chain()->valid(&reason)) {
 107                 throw InvalidSignerError (reason);
 108         }
 109 }
 110
 111
 112 void
 113 Writer::start ()
 114 {
 115         if (!_text_only) {
 116                 _thread = boost::thread (boost::bind(&Writer::thread, this));
 117 #ifdef DCPOMATIC_LINUX
 118                 pthread_setname_np (_thread.native_handle(), "writer");
 119 #endif
 120         }
 121 }
 122
 123
 124 Writer::~Writer ()
 125 {
 126         if (!_text_only) {
 127                 terminate_thread (false);
 128         }
 129 }
 130
 131
 132 /** Pass a video frame to the writer for writing to disk at some point.
 133  *  This method can be called with frames out of order.
 134  *  @param encoded JPEG2000-encoded data.
 135  *  @param frame Frame index within the DCP.
 136  *  @param eyes Eyes that this frame image is for.
 137  */
 138 void
 139 Writer::write (shared_ptr<const Data> encoded, Frame frame, Eyes eyes)
 140 {
 141         boost::mutex::scoped_lock lock (_state_mutex);
 142
 143         while (_queued_full_in_memory > _maximum_frames_in_memory) {
 144                 /* There are too many full frames in memory; wake the main writer thread and
 145                    wait until it sorts everything out */
 146                 _empty_condition.notify_all ();
 147                 _full_condition.wait (lock);
 148         }
 149
 150         QueueItem qi;
 151         qi.type = QueueItem::Type::FULL;
 152         qi.encoded = encoded;
 153         qi.reel = video_reel (frame);
 154         qi.frame = frame - _reels[qi.reel].start ();
 155
 156         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 157
 158         qi.eyes = eyes;
 159         _queue.push_back(qi);
 160         ++_queued_full_in_memory;
 161
 162         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 163         _empty_condition.notify_all ();
 164 }
 165
 166
 167 bool
 168 Writer::can_repeat (Frame frame) const
 169 {
 170         return frame > _reels[video_reel(frame)].start();
 171 }
 172
 173
 174 /** Repeat the last frame that was written to a reel as a new frame.
 175  *  @param frame Frame index within the DCP of the new (repeated) frame.
 176  *  @param eyes Eyes that this repeated frame image is for.
 177  */
 178 void
 179 Writer::repeat (Frame frame, Eyes eyes)
 180 {
 181         boost::mutex::scoped_lock lock (_state_mutex);
 182
 183         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 184                 /* The queue is too big, and the main writer thread can run and fix it, so
 185                    wake it and wait until it has done.
 186                 */
 187                 _empty_condition.notify_all ();
 188                 _full_condition.wait (lock);
 189         }
 190
 191         QueueItem qi;
 192         qi.type = QueueItem::Type::REPEAT;
 193         qi.reel = video_reel (frame);
 194         qi.frame = frame - _reels[qi.reel].start ();
 195         if (film()->three_d() && eyes == Eyes::BOTH) {
 196                 qi.eyes = Eyes::LEFT;
 197                 _queue.push_back (qi);
 198                 qi.eyes = Eyes::RIGHT;
 199                 _queue.push_back (qi);
 200         } else {
 201                 qi.eyes = eyes;
 202                 _queue.push_back (qi);
 203         }
 204
 205         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 206         _empty_condition.notify_all ();
 207 }
 208
 209
 210 void
 211 Writer::fake_write (Frame frame, Eyes eyes)
 212 {
 213         boost::mutex::scoped_lock lock (_state_mutex);
 214
 215         while (_queue.size() > _maximum_queue_size && have_sequenced_image_at_queue_head()) {
 216                 /* The queue is too big, and the main writer thread can run and fix it, so
 217                    wake it and wait until it has done.
 218                 */
 219                 _empty_condition.notify_all ();
 220                 _full_condition.wait (lock);
 221         }
 222
 223         size_t const reel = video_reel (frame);
 224         Frame const frame_in_reel = frame - _reels[reel].start ();
 225
 226         QueueItem qi;
 227         qi.type = QueueItem::Type::FAKE;
 228
 229         {
 230                 shared_ptr<InfoFileHandle> info_file = film()->info_file_handle(_reels[reel].period(), true);
 231                 qi.size = _reels[reel].read_frame_info(info_file, frame_in_reel, eyes).size;
 232         }
 233
 234         DCPOMATIC_ASSERT((film()->three_d() && eyes != Eyes::BOTH) || (!film()->three_d() && eyes == Eyes::BOTH));
 235
 236         qi.reel = reel;
 237         qi.frame = frame_in_reel;
 238         qi.eyes = eyes;
 239         _queue.push_back(qi);
 240
 241         /* Now there's something to do: wake anything wait()ing on _empty_condition */
 242         _empty_condition.notify_all ();
 243 }
 244
 245
 246 /** Write some audio frames to the DCP.
 247  *  @param audio Audio data.
 248  *  @param time Time of this data within the DCP.
 249  *  This method is not thread safe.
 250  */
 251 void
 252 Writer::write (shared_ptr<const AudioBuffers> audio, DCPTime const time)
 253 {
 254         DCPOMATIC_ASSERT (audio);
 255
 256         int const afr = film()->audio_frame_rate();
 257
 258         DCPTime const end = time + DCPTime::from_frames(audio->frames(), afr);
 259
 260         /* The audio we get might span a reel boundary, and if so we have to write it in bits */
 261
 262         DCPTime t = time;
 263         while (t < end) {
 264
 265                 if (_audio_reel == _reels.end ()) {
 266                         /* This audio is off the end of the last reel; ignore it */
 267                         return;
 268                 }
 269
 270                 if (end <= _audio_reel->period().to) {
 271                         /* Easy case: we can write all the audio to this reel */
 272                         _audio_reel->write (audio);
 273                         t = end;
 274                 } else if (_audio_reel->period().to <= t) {
 275                         /* This reel is entirely before the start of our audio; just skip the reel */
 276                         ++_audio_reel;
 277                 } else {
 278                         /* This audio is over a reel boundary; split the audio into two and write the first part */
 279                         DCPTime part_lengths[2] = {
 280                                 _audio_reel->period().to - t,
 281                                 end - _audio_reel->period().to
 282                         };
 283
 284                         /* Be careful that part_lengths[0] + part_lengths[1] can't be bigger than audio->frames() */
 285                         Frame part_frames[2] = {
 286                                 part_lengths[0].frames_ceil(afr),
 287                                 part_lengths[1].frames_floor(afr)
 288                         };
 289
 290                         DCPOMATIC_ASSERT ((part_frames[0] + part_frames[1]) <= audio->frames());
 291
 292                         if (part_frames[0]) {
 293                                 auto part = make_shared<AudioBuffers>(audio, part_frames[0], 0);
 294                                 _audio_reel->write (part);
 295                         }
 296
 297                         if (part_frames[1]) {
 298                                 audio = make_shared<AudioBuffers>(audio, part_frames[1], part_frames[0]);
 299                         } else {
 300                                 audio.reset ();
 301                         }
 302
 303                         ++_audio_reel;
 304                         t += part_lengths[0];
 305                 }
 306         }
 307 }
 308
 309
 310 void
 311 Writer::write (shared_ptr<const dcp::AtmosFrame> atmos, DCPTime time, AtmosMetadata metadata)
 312 {
 313         if (_atmos_reel->period().to == time) {
 314                 ++_atmos_reel;
 315                 DCPOMATIC_ASSERT (_atmos_reel != _reels.end());
 316         }
 317
 318         /* We assume that we get a video frame's worth of data here */
 319         _atmos_reel->write (atmos, metadata);
 320 }
 321
 322
 323 /** Caller must hold a lock on _state_mutex */
 324 bool
 325 Writer::have_sequenced_image_at_queue_head ()
 326 {
 327         if (_queue.empty ()) {
 328                 return false;
 329         }
 330
 331         _queue.sort ();
 332         auto const & f = _queue.front();
 333         return _last_written[f.reel].next(f);
 334 }
 335
 336
 337 bool
 338 Writer::LastWritten::next (QueueItem qi) const
 339 {
 340         if (qi.eyes == Eyes::BOTH) {
 341                 /* 2D */
 342                 return qi.frame == (_frame + 1);
 343         }
 344
 345         /* 3D */
 346
 347         if (_eyes == Eyes::LEFT && qi.frame == _frame && qi.eyes == Eyes::RIGHT) {
 348                 return true;
 349         }
 350
 351         if (_eyes == Eyes::RIGHT && qi.frame == (_frame + 1) && qi.eyes == Eyes::LEFT) {
 352                 return true;
 353         }
 354
 355         return false;
 356 }
 357
 358
 359 void
 360 Writer::LastWritten::update (QueueItem qi)
 361 {
 362         _frame = qi.frame;
 363         _eyes = qi.eyes;
 364 }
 365
 366
 367 void
 368 Writer::thread ()
 369 try
 370 {
 371         start_of_thread ("Writer");
 372
 373         while (true)
 374         {
 375                 boost::mutex::scoped_lock lock (_state_mutex);
 376
 377                 while (true) {
 378
 379                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 380                                 /* We've got something to do: go and do it */
 381                                 break;
 382                         }
 383
 384                         /* Nothing to do: wait until something happens which may indicate that we do */
 385                         LOG_TIMING (N_("writer-sleep queue=%1"), _queue.size());
 386                         _empty_condition.wait (lock);
 387                         LOG_TIMING (N_("writer-wake queue=%1"), _queue.size());
 388                 }
 389
 390                 /* We stop here if we have been asked to finish, and if either the queue
 391                    is empty or we do not have a sequenced image at its head (if this is the
 392                    case we will never terminate as no new frames will be sent once
 393                    _finish is true).
 394                 */
 395                 if (_finish && (!have_sequenced_image_at_queue_head() || _queue.empty())) {
 396                         /* (Hopefully temporarily) log anything that was not written */
 397                         if (!_queue.empty() && !have_sequenced_image_at_queue_head()) {
 398                                 LOG_WARNING (N_("Finishing writer with a left-over queue of %1:"), _queue.size());
 399                                 for (auto const& i: _queue) {
 400                                         if (i.type == QueueItem::Type::FULL) {
 401                                                 LOG_WARNING (N_("- type FULL, frame %1, eyes %2"), i.frame, (int) i.eyes);
 402                                         } else {
 403                                                 LOG_WARNING (N_("- type FAKE, size %1, frame %2, eyes %3"), i.size, i.frame, (int) i.eyes);
 404                                         }
 405                                 }
 406                         }
 407                         return;
 408                 }
 409
 410                 /* Write any frames that we can write; i.e. those that are in sequence. */
 411                 while (have_sequenced_image_at_queue_head ()) {
 412                         auto qi = _queue.front ();
 413                         _last_written[qi.reel].update (qi);
 414                         _queue.pop_front ();
 415                         if (qi.type == QueueItem::Type::FULL && qi.encoded) {
 416                                 --_queued_full_in_memory;
 417                         }
 418
 419                         lock.unlock ();
 420
 421                         auto& reel = _reels[qi.reel];
 422
 423                         switch (qi.type) {
 424                         case QueueItem::Type::FULL:
 425                                 LOG_DEBUG_ENCODE (N_("Writer FULL-writes %1 (%2)"), qi.frame, (int) qi.eyes);
 426                                 if (!qi.encoded) {
 427                                         qi.encoded.reset (new ArrayData(film()->j2c_path(qi.reel, qi.frame, qi.eyes, false)));
 428                                 }
 429                                 reel.write (qi.encoded, qi.frame, qi.eyes);
 430                                 ++_full_written;
 431                                 break;
 432                         case QueueItem::Type::FAKE:
 433                                 LOG_DEBUG_ENCODE (N_("Writer FAKE-writes %1"), qi.frame);
 434                                 reel.fake_write (qi.size);
 435                                 ++_fake_written;
 436                                 break;
 437                         case QueueItem::Type::REPEAT:
 438                                 LOG_DEBUG_ENCODE (N_("Writer REPEAT-writes %1"), qi.frame);
 439                                 reel.repeat_write (qi.frame, qi.eyes);
 440                                 ++_repeat_written;
 441                                 break;
 442                         }
 443
 444                         lock.lock ();
 445                         _full_condition.notify_all ();
 446                 }
 447
 448                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 449                         /* Too many frames in memory which can't yet be written to the stream.
 450                            Write some FULL frames to disk.
 451                         */
 452
 453                         /* Find one from the back of the queue */
 454                         _queue.sort ();
 455                         auto i = _queue.rbegin ();
 456                         while (i != _queue.rend() && (i->type != QueueItem::Type::FULL || !i->encoded)) {
 457                                 ++i;
 458                         }
 459
 460                         DCPOMATIC_ASSERT (i != _queue.rend());
 461                         ++_pushed_to_disk;
 462                         /* For the log message below */
 463                         int const awaiting = _last_written[_queue.front().reel].frame() + 1;
 464                         lock.unlock ();
 465
 466                         /* i is valid here, even though we don't hold a lock on the mutex,
 467                            since list iterators are unaffected by insertion and only this
 468                            thread could erase the last item in the list.
 469                         */
 470
 471                         LOG_GENERAL ("Writer full; pushes %1 to disk while awaiting %2", i->frame, awaiting);
 472
 473                         i->encoded->write_via_temp (
 474                                 film()->j2c_path(i->reel, i->frame, i->eyes, true),
 475                                 film()->j2c_path(i->reel, i->frame, i->eyes, false)
 476                                 );
 477
 478                         lock.lock ();
 479                         i->encoded.reset ();
 480                         --_queued_full_in_memory;
 481                         _full_condition.notify_all ();
 482                 }
 483         }
 484 }
 485 catch (...)
 486 {
 487         store_current ();
 488 }
 489
 490
 491 void
 492 Writer::terminate_thread (bool can_throw)
 493 {
 494         boost::this_thread::disable_interruption dis;
 495
 496         boost::mutex::scoped_lock lock (_state_mutex);
 497
 498         _finish = true;
 499         _empty_condition.notify_all ();
 500         _full_condition.notify_all ();
 501         lock.unlock ();
 502
 503         try {
 504                 _thread.join ();
 505         } catch (...) {}
 506
 507         if (can_throw) {
 508                 rethrow ();
 509         }
 510 }
 511
 512
 513 void
 514 Writer::calculate_digests ()
 515 {
 516         auto job = _job.lock ();
 517         if (job) {
 518                 job->sub (_("Computing digests"));
 519         }
 520
 521         boost::asio::io_service service;
 522         boost::thread_group pool;
 523
 524         auto work = make_shared<boost::asio::io_service::work>(service);
 525
 526         int const threads = max (1, Config::instance()->master_encoding_threads());
 527
 528         for (int i = 0; i < threads; ++i) {
 529                 pool.create_thread (boost::bind (&boost::asio::io_service::run, &service));
 530         }
 531
 532         std::function<void (float)> set_progress;
 533         if (job) {
 534                 set_progress = boost::bind (&Writer::set_digest_progress, this, job.get(), _1);
 535         } else {
 536                 set_progress = [](float) {
 537                         boost::this_thread::interruption_point();
 538                 };
 539         }
 540
 541         for (auto& i: _reels) {
 542                 service.post (boost::bind (&ReelWriter::calculate_digests, &i, set_progress));
 543         }
 544         service.post (boost::bind (&Writer::calculate_referenced_digests, this, set_progress));
 545
 546         work.reset ();
 547
 548         try {
 549                 pool.join_all ();
 550         } catch (boost::thread_interrupted) {
 551                 /* join_all was interrupted, so we need to interrupt the threads
 552                  * in our pool then try again to join them.
 553                  */
 554                 pool.interrupt_all ();
 555                 pool.join_all ();
 556         }
 557
 558         service.stop ();
 559 }
 560
 561
 562 /** @param output_dcp Path to DCP folder to write */
 563 void
 564 Writer::finish (boost::filesystem::path output_dcp)
 565 {
 566         if (_thread.joinable()) {
 567                 LOG_GENERAL_NC ("Terminating writer thread");
 568                 terminate_thread (true);
 569         }
 570
 571         LOG_GENERAL_NC ("Finishing ReelWriters");
 572
 573         for (auto& i: _reels) {
 574                 write_hanging_text (i);
 575                 i.finish (output_dcp);
 576         }
 577
 578         LOG_GENERAL_NC ("Writing XML");
 579
 580         dcp::DCP dcp (output_dcp);
 581
 582         auto cpl = make_shared<dcp::CPL>(
 583                 film()->dcp_name(),
 584                 film()->dcp_content_type()->libdcp_kind(),
 585                 film()->interop() ? dcp::Standard::INTEROP : dcp::Standard::SMPTE
 586                 );
 587
 588         dcp.add (cpl);
 589
 590         calculate_digests ();
 591
 592         /* Add reels */
 593
 594         for (auto& i: _reels) {
 595                 cpl->add (i.create_reel(_reel_assets, _fonts, _chosen_interop_font, output_dcp, _have_subtitles, _have_closed_captions));
 596         }
 597
 598         /* Add metadata */
 599
 600         auto creator = Config::instance()->dcp_creator();
 601         if (creator.empty()) {
 602                 creator = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 603         }
 604
 605         auto issuer = Config::instance()->dcp_issuer();
 606         if (issuer.empty()) {
 607                 issuer = String::compose("DCP-o-matic %1 %2", dcpomatic_version, dcpomatic_git_commit);
 608         }
 609
 610         cpl->set_creator (creator);
 611         cpl->set_issuer (issuer);
 612
 613         cpl->set_ratings (film()->ratings());
 614
 615         vector<dcp::ContentVersion> cv;
 616         for (auto i: film()->content_versions()) {
 617                 cv.push_back (dcp::ContentVersion(i));
 618         }
 619         if (cv.empty()) {
 620                 cv = { dcp::ContentVersion("1") };
 621         }
 622         cpl->set_content_versions (cv);
 623
 624         cpl->set_full_content_title_text (film()->name());
 625         cpl->set_full_content_title_text_language (film()->name_language());
 626         if (film()->release_territory()) {
 627                 cpl->set_release_territory (*film()->release_territory());
 628         }
 629         cpl->set_version_number (film()->version_number());
 630         cpl->set_status (film()->status());
 631         if (film()->chain()) {
 632                 cpl->set_chain (*film()->chain());
 633         }
 634         if (film()->distributor()) {
 635                 cpl->set_distributor (*film()->distributor());
 636         }
 637         if (film()->facility()) {
 638                 cpl->set_facility (*film()->facility());
 639         }
 640         if (film()->luminance()) {
 641                 cpl->set_luminance (*film()->luminance());
 642         }
 643         if (film()->sign_language_video_language()) {
 644                 cpl->set_sign_language_video_language (*film()->sign_language_video_language());
 645         }
 646
 647         dcp::MCASoundField field;
 648         if (film()->audio_channels() == 2) {
 649                 field = dcp::MCASoundField::STEREO;
 650         } else if (film()->audio_channels() <= 6) {
 651                 field = dcp::MCASoundField::FIVE_POINT_ONE;
 652         } else {
 653                 field = dcp::MCASoundField::SEVEN_POINT_ONE;
 654         }
 655
 656         dcp::MainSoundConfiguration msc (field, film()->audio_channels());
 657         for (auto i: film()->mapped_audio_channels()) {
 658                 if (static_cast<int>(i) < film()->audio_channels()) {
 659                         msc.set_mapping (i, static_cast<dcp::Channel>(i));
 660                 }
 661         }
 662
 663         cpl->set_main_sound_configuration (msc.to_string());
 664         cpl->set_main_sound_sample_rate (film()->audio_frame_rate());
 665         cpl->set_main_picture_stored_area (film()->frame_size());
 666
 667         auto active_area = film()->active_area();
 668         if (active_area.width > 0 && active_area.height > 0) {
 669                 /* It's not allowed to have a zero active area width or height, and the sizes must be multiples of 2 */
 670                 cpl->set_main_picture_active_area({ active_area.width & ~1, active_area.height & ~1});
 671         }
 672
 673         auto sl = film()->subtitle_languages().second;
 674         if (!sl.empty()) {
 675                 cpl->set_additional_subtitle_languages(sl);
 676         }
 677
 678         auto signer = Config::instance()->signer_chain();
 679         /* We did check earlier, but check again here to be on the safe side */
 680         string reason;
 681         if (!signer->valid (&reason)) {
 682                 throw InvalidSignerError (reason);
 683         }
 684
 685         dcp.set_issuer(issuer);
 686         dcp.set_creator(creator);
 687         dcp.set_annotation_text(film()->dcp_name());
 688
 689         dcp.write_xml(signer, !film()->limit_to_smpte_bv20(), Config::instance()->dcp_metadata_filename_format());
 690
 691         LOG_GENERAL (
 692                 N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT, %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk
 693                 );
 694
 695         write_cover_sheet (output_dcp);
 696 }
 697
 698
 699 void
 700 Writer::write_cover_sheet (boost::filesystem::path output_dcp)
 701 {
 702         auto const cover = film()->file("COVER_SHEET.txt");
 703         dcp::File f(cover, "w");
 704         if (!f) {
 705                 throw OpenFileError (cover, errno, OpenFileError::WRITE);
 706         }
 707
 708         auto text = Config::instance()->cover_sheet ();
 709         boost::algorithm::replace_all (text, "$CPL_NAME", film()->name());
 710         auto cpls = film()->cpls();
 711         if (!cpls.empty()) {
 712                 boost::algorithm::replace_all (text, "$CPL_FILENAME", cpls[0].cpl_file.filename().string());
 713         }
 714         boost::algorithm::replace_all (text, "$TYPE", film()->dcp_content_type()->pretty_name());
 715         boost::algorithm::replace_all (text, "$CONTAINER", film()->container()->container_nickname());
 716
 717         auto audio_language = film()->audio_language();
 718         if (audio_language) {
 719                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", audio_language->description());
 720         } else {
 721                 boost::algorithm::replace_all (text, "$AUDIO_LANGUAGE", _("None"));
 722         }
 723
 724         auto subtitle_languages = film()->subtitle_languages();
 725         if (subtitle_languages.first) {
 726                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", subtitle_languages.first->description());
 727         } else {
 728                 boost::algorithm::replace_all (text, "$SUBTITLE_LANGUAGE", _("None"));
 729         }
 730
 731         boost::uintmax_t size = 0;
 732         for (
 733                 auto i = boost::filesystem::recursive_directory_iterator(output_dcp);
 734                 i != boost::filesystem::recursive_directory_iterator();
 735                 ++i) {
 736                 if (boost::filesystem::is_regular_file (i->path())) {
 737                         size += boost::filesystem::file_size (i->path());
 738                 }
 739         }
 740
 741         if (size > (1000000000L)) {
 742                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1GB", dcp::locale_convert<string>(size / 1000000000.0, 1, true)));
 743         } else {
 744                 boost::algorithm::replace_all (text, "$SIZE", String::compose("%1MB", dcp::locale_convert<string>(size / 1000000.0, 1, true)));
 745         }
 746
 747         auto ch = audio_channel_types (film()->mapped_audio_channels(), film()->audio_channels());
 748         auto description = String::compose("%1.%2", ch.first, ch.second);
 749
 750         if (description == "0.0") {
 751                 description = _("None");
 752         } else if (description == "1.0") {
 753                 description = _("Mono");
 754         } else if (description == "2.0") {
 755                 description = _("Stereo");
 756         }
 757         boost::algorithm::replace_all (text, "$AUDIO", description);
 758
 759         auto const hmsf = film()->length().split(film()->video_frame_rate());
 760         string length;
 761         if (hmsf.h == 0 && hmsf.m == 0) {
 762                 length = String::compose("%1s", hmsf.s);
 763         } else if (hmsf.h == 0 && hmsf.m > 0) {
 764                 length = String::compose("%1m%2s", hmsf.m, hmsf.s);
 765         } else if (hmsf.h > 0 && hmsf.m > 0) {
 766                 length = String::compose("%1h%2m%3s", hmsf.h, hmsf.m, hmsf.s);
 767         }
 768
 769         boost::algorithm::replace_all (text, "$LENGTH", length);
 770
 771         f.checked_write(text.c_str(), text.length());
 772 }
 773
 774
 775 /** @param frame Frame index within the whole DCP.
 776  *  @return true if we can fake-write this frame.
 777  */
 778 bool
 779 Writer::can_fake_write (Frame frame) const
 780 {
 781         if (film()->encrypted()) {
 782                 /* We need to re-write the frame because the asset ID is embedded in the HMAC... I think... */
 783                 return false;
 784         }
 785
 786         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 787            parameters in the asset writer.
 788         */
 789
 790         auto const & reel = _reels[video_reel(frame)];
 791
 792         /* Make frame relative to the start of the reel */
 793         frame -= reel.start ();
 794         return (frame != 0 && frame < reel.first_nonexistent_frame());
 795 }
 796
 797
 798 /** @param track Closed caption track if type == TextType::CLOSED_CAPTION */
 799 void
 800 Writer::write (PlayerText text, TextType type, optional<DCPTextTrack> track, DCPTimePeriod period)
 801 {
 802         vector<ReelWriter>::iterator* reel = nullptr;
 803
 804         switch (type) {
 805         case TextType::OPEN_SUBTITLE:
 806                 reel = &_subtitle_reel;
 807                 _have_subtitles = true;
 808                 break;
 809         case TextType::CLOSED_CAPTION:
 810                 DCPOMATIC_ASSERT (track);
 811                 DCPOMATIC_ASSERT (_caption_reels.find(*track) != _caption_reels.end());
 812                 reel = &_caption_reels[*track];
 813                 _have_closed_captions.insert (*track);
 814                 break;
 815         default:
 816                 DCPOMATIC_ASSERT (false);
 817         }
 818
 819         DCPOMATIC_ASSERT (*reel != _reels.end());
 820         while ((*reel)->period().to <= period.from) {
 821                 ++(*reel);
 822                 DCPOMATIC_ASSERT (*reel != _reels.end());
 823                 write_hanging_text (**reel);
 824         }
 825
 826         auto back_off = [this](DCPTimePeriod period) {
 827                 period.to -= DCPTime::from_frames(2, film()->video_frame_rate());
 828                 return period;
 829         };
 830
 831         if (period.to > (*reel)->period().to) {
 832                 /* This text goes off the end of the reel.  Store parts of it that should go into
 833                  * other reels.
 834                  */
 835                 for (auto i = std::next(*reel); i != _reels.end(); ++i) {
 836                         auto overlap = i->period().overlap(period);
 837                         if (overlap) {
 838                                 _hanging_texts.push_back (HangingText{text, type, track, back_off(*overlap)});
 839                         }
 840                 }
 841                 /* Back off from the reel boundary by a couple of frames to avoid tripping checks
 842                  * for subtitles being too close together.
 843                  */
 844                 period.to = (*reel)->period().to;
 845                 period = back_off(period);
 846         }
 847
 848         (*reel)->write(text, type, track, period, _fonts);
 849 }
 850
 851
 852 void
 853 Writer::write (vector<shared_ptr<Font>> fonts)
 854 {
 855         if (fonts.empty()) {
 856                 return;
 857         }
 858
 859         /* Fonts may come in with empty IDs but we don't want to put those in the DCP */
 860         auto fix_id = [](string id) {
 861                 return id.empty() ? "font" : id;
 862         };
 863
 864         if (film()->interop()) {
 865                 /* Interop will ignore second and subsequent <LoadFont>s so we don't want to
 866                  * even write them as they upset some validators.  Set up _fonts so that every
 867                  * font used by any subtitle will be written with the same ID.
 868                  */
 869                 for (size_t i = 0; i < fonts.size(); ++i) {
 870                         _fonts.put(fonts[i], fix_id(fonts[0]->id()));
 871                 }
 872                 _chosen_interop_font = fonts[0];
 873         } else {
 874                 set<string> used_ids;
 875
 876                 /* Return the index of a _N at the end of a string, or string::npos */
 877                 auto underscore_number_position = [](string s) {
 878                         auto last_underscore = s.find_last_of("_");
 879                         if (last_underscore == string::npos) {
 880                                 return string::npos;
 881                         }
 882
 883                         for (auto i = last_underscore + 1; i < s.size(); ++i) {
 884                                 if (!isdigit(s[i])) {
 885                                         return string::npos;
 886                                 }
 887                         }
 888
 889                         return last_underscore;
 890                 };
 891
 892                 /* Write fonts to _fonts, changing any duplicate IDs so that they are unique */
 893                 for (auto font: fonts) {
 894                         auto id = fix_id(font->id());
 895                         if (used_ids.find(id) == used_ids.end()) {
 896                                 /* This ID is unique so we can just use it as-is */
 897                                 _fonts.put(font, id);
 898                                 used_ids.insert(id);
 899                         } else {
 900                                 auto end = underscore_number_position(id);
 901                                 if (end == string::npos) {
 902                                         /* This string has no _N suffix, so add one */
 903                                         id += "_0";
 904                                         end = underscore_number_position(id);
 905                                 }
 906
 907                                 ++end;
 908
 909                                 /* Increment the suffix until we find a unique one */
 910                                 auto number = dcp::raw_convert<int>(id.substr(end));
 911                                 while (used_ids.find(id) != used_ids.end()) {
 912                                         ++number;
 913                                         id = String::compose("%1_%2", id.substr(0, end - 1), number);
 914                                 }
 915                                 used_ids.insert(id);
 916                         }
 917                         _fonts.put(font, id);
 918                 }
 919
 920                 DCPOMATIC_ASSERT(_fonts.map().size() == used_ids.size());
 921         }
 922 }
 923
 924
 925 bool
 926 operator< (QueueItem const & a, QueueItem const & b)
 927 {
 928         if (a.reel != b.reel) {
 929                 return a.reel < b.reel;
 930         }
 931
 932         if (a.frame != b.frame) {
 933                 return a.frame < b.frame;
 934         }
 935
 936         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 937 }
 938
 939
 940 bool
 941 operator== (QueueItem const & a, QueueItem const & b)
 942 {
 943         return a.reel == b.reel && a.frame == b.frame && a.eyes == b.eyes;
 944 }
 945
 946
 947 void
 948 Writer::set_encoder_threads (int threads)
 949 {
 950         boost::mutex::scoped_lock lm (_state_mutex);
 951         _maximum_frames_in_memory = lrint (threads * Config::instance()->frames_in_memory_multiplier());
 952         _maximum_queue_size = threads * 16;
 953 }
 954
 955
 956 void
 957 Writer::write (ReferencedReelAsset asset)
 958 {
 959         _reel_assets.push_back (asset);
 960 }
 961
 962
 963 size_t
 964 Writer::video_reel (int frame) const
 965 {
 966         auto t = DCPTime::from_frames (frame, film()->video_frame_rate());
 967         size_t i = 0;
 968         while (i < _reels.size() && !_reels[i].period().contains (t)) {
 969                 ++i;
 970         }
 971
 972         DCPOMATIC_ASSERT (i < _reels.size ());
 973         return i;
 974 }
 975
 976
 977 void
 978 Writer::set_digest_progress (Job* job, float progress)
 979 {
 980         boost::mutex::scoped_lock lm (_digest_progresses_mutex);
 981
 982         _digest_progresses[boost::this_thread::get_id()] = progress;
 983         float min_progress = FLT_MAX;
 984         for (auto const& i: _digest_progresses) {
 985                 min_progress = min (min_progress, i.second);
 986         }
 987
 988         job->set_progress (min_progress);
 989
 990         Waker waker;
 991         waker.nudge ();
 992
 993         boost::this_thread::interruption_point();
 994 }
 995
 996
 997 /** Calculate hashes for any referenced MXF assets which do not already have one */
 998 void
 999 Writer::calculate_referenced_digests (std::function<void (float)> set_progress)
1000 try
1001 {
1002         for (auto const& i: _reel_assets) {
1003                 auto file = dynamic_pointer_cast<dcp::ReelFileAsset>(i.asset);
1004                 if (file && !file->hash()) {
1005                         file->asset_ref().asset()->hash (set_progress);
1006                         file->set_hash (file->asset_ref().asset()->hash());
1007                 }
1008         }
1009 } catch (boost::thread_interrupted) {
1010         /* set_progress contains an interruption_point, so any of these methods
1011          * may throw thread_interrupted, at which point we just give up.
1012          */
1013 }
1014
1015
1016 void
1017 Writer::write_hanging_text (ReelWriter& reel)
1018 {
1019         vector<HangingText> new_hanging_texts;
1020         for (auto i: _hanging_texts) {
1021                 if (i.period.from == reel.period().from) {
1022                         reel.write (i.text, i.type, i.track, i.period, _fonts);
1023                 } else {
1024                         new_hanging_texts.push_back (i);
1025                 }
1026         }
1027         _hanging_texts = new_hanging_texts;
1028 }