src/lib/writer.cc

   1 /*
   2     Copyright (C) 2012 Carl Hetherington <cth@carlh.net>
   3
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation; either version 2 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program; if not, write to the Free Software
  16     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17
  18 */
  19
  20 #include <fstream>
  21 #include <cerrno>
  22 #include <libdcp/mono_picture_asset.h>
  23 #include <libdcp/stereo_picture_asset.h>
  24 #include <libdcp/sound_asset.h>
  25 #include <libdcp/reel.h>
  26 #include <libdcp/dcp.h>
  27 #include <libdcp/cpl.h>
  28 #include "writer.h"
  29 #include "compose.hpp"
  30 #include "film.h"
  31 #include "ratio.h"
  32 #include "log.h"
  33 #include "dcp_video_frame.h"
  34 #include "dcp_content_type.h"
  35 #include "player.h"
  36 #include "audio_mapping.h"
  37 #include "config.h"
  38 #include "job.h"
  39 #include "cross.h"
  40
  41 #include "i18n.h"
  42
  43 using std::make_pair;
  44 using std::pair;
  45 using std::string;
  46 using std::list;
  47 using std::cout;
  48 using boost::shared_ptr;
  49 using boost::weak_ptr;
  50
  51 int const Writer::_maximum_frames_in_memory = Config::instance()->num_local_encoding_threads() + 4;
  52
  53 Writer::Writer (shared_ptr<const Film> f, weak_ptr<Job> j)
  54         : _film (f)
  55         , _job (j)
  56         , _first_nonexistant_frame (0)
  57         , _thread (0)
  58         , _finish (false)
  59         , _queued_full_in_memory (0)
  60         , _last_written_frame (-1)
  61         , _last_written_eyes (EYES_RIGHT)
  62         , _full_written (0)
  63         , _fake_written (0)
  64         , _repeat_written (0)
  65         , _pushed_to_disk (0)
  66 {
  67         /* Remove any old DCP */
  68         boost::filesystem::remove_all (_film->dir (_film->dcp_name ()));
  69
  70         shared_ptr<Job> job = _job.lock ();
  71         assert (job);
  72
  73         job->sub (_("Checking existing image data"));
  74         check_existing_picture_mxf ();
  75
  76         /* Create our picture asset in a subdirectory, named according to those
  77            film's parameters which affect the video output.  We will hard-link
  78            it into the DCP later.
  79         */
  80
  81         if (_film->three_d ()) {
  82                 _picture_asset.reset (new libdcp::StereoPictureAsset (_film->internal_video_mxf_dir (), _film->internal_video_mxf_filename ()));
  83         } else {
  84                 _picture_asset.reset (new libdcp::MonoPictureAsset (_film->internal_video_mxf_dir (), _film->internal_video_mxf_filename ()));
  85         }
  86
  87         _picture_asset->set_edit_rate (_film->video_frame_rate ());
  88         _picture_asset->set_size (fit_ratio_within (_film->container()->ratio(), _film->full_frame ()));
  89
  90         if (_film->encrypted ()) {
  91                 _picture_asset->set_key (_film->key ());
  92         }
  93
  94         _picture_asset_writer = _picture_asset->start_write (_first_nonexistant_frame > 0);
  95
  96         /* Write the sound asset into the film directory so that we leave the creation
  97            of the DCP directory until the last minute.  Some versions of windows inexplicably
  98            don't like overwriting existing files here, so try to remove it using boost.
  99         */
 100         boost::system::error_code ec;
 101         boost::filesystem::remove (_film->file (_film->audio_mxf_filename ()), ec);
 102         if (ec) {
 103                 _film->log()->log (String::compose ("Could not remove existing audio MXF file (%1)", ec.value ()));
 104         }
 105
 106         _sound_asset.reset (new libdcp::SoundAsset (_film->directory (), _film->audio_mxf_filename ()));
 107         _sound_asset->set_edit_rate (_film->video_frame_rate ());
 108         _sound_asset->set_channels (_film->audio_channels ());
 109         _sound_asset->set_sampling_rate (_film->audio_frame_rate ());
 110
 111         if (_film->encrypted ()) {
 112                 _sound_asset->set_key (_film->key ());
 113         }
 114
 115         _sound_asset_writer = _sound_asset->start_write ();
 116
 117         _thread = new boost::thread (boost::bind (&Writer::thread, this));
 118
 119         job->sub (_("Encoding image data"));
 120 }
 121
 122 void
 123 Writer::write (shared_ptr<const EncodedData> encoded, int frame, Eyes eyes)
 124 {
 125         boost::mutex::scoped_lock lock (_mutex);
 126
 127         QueueItem qi;
 128         qi.type = QueueItem::FULL;
 129         qi.encoded = encoded;
 130         qi.frame = frame;
 131
 132         if (_film->three_d() && eyes == EYES_BOTH) {
 133                 /* 2D material in a 3D DCP; fake the 3D */
 134                 qi.eyes = EYES_LEFT;
 135                 _queue.push_back (qi);
 136                 ++_queued_full_in_memory;
 137                 qi.eyes = EYES_RIGHT;
 138                 _queue.push_back (qi);
 139                 ++_queued_full_in_memory;
 140         } else {
 141                 qi.eyes = eyes;
 142                 _queue.push_back (qi);
 143                 ++_queued_full_in_memory;
 144         }
 145
 146         _condition.notify_all ();
 147 }
 148
 149 void
 150 Writer::fake_write (int frame, Eyes eyes)
 151 {
 152         boost::mutex::scoped_lock lock (_mutex);
 153
 154         FILE* ifi = fopen_boost (_film->info_path (frame, eyes), "r");
 155         libdcp::FrameInfo info (ifi);
 156         fclose (ifi);
 157
 158         QueueItem qi;
 159         qi.type = QueueItem::FAKE;
 160         qi.size = info.size;
 161         qi.frame = frame;
 162         if (_film->three_d() && eyes == EYES_BOTH) {
 163                 qi.eyes = EYES_LEFT;
 164                 _queue.push_back (qi);
 165                 qi.eyes = EYES_RIGHT;
 166                 _queue.push_back (qi);
 167         } else {
 168                 qi.eyes = eyes;
 169                 _queue.push_back (qi);
 170         }
 171
 172         _condition.notify_all ();
 173 }
 174
 175 /** This method is not thread safe */
 176 void
 177 Writer::write (shared_ptr<const AudioBuffers> audio)
 178 {
 179         _sound_asset_writer->write (audio->data(), audio->frames());
 180 }
 181
 182 /** This must be called from Writer::thread() with an appropriate lock held */
 183 bool
 184 Writer::have_sequenced_image_at_queue_head ()
 185 {
 186         if (_queue.empty ()) {
 187                 return false;
 188         }
 189
 190         _queue.sort ();
 191
 192         /* The queue should contain only EYES_LEFT/EYES_RIGHT pairs or EYES_BOTH */
 193
 194         if (_queue.front().eyes == EYES_BOTH) {
 195                 /* 2D */
 196                 return _queue.front().frame == (_last_written_frame + 1);
 197         }
 198
 199         /* 3D */
 200
 201         if (_last_written_eyes == EYES_LEFT && _queue.front().frame == _last_written_frame && _queue.front().eyes == EYES_RIGHT) {
 202                 return true;
 203         }
 204
 205         if (_last_written_eyes == EYES_RIGHT && _queue.front().frame == (_last_written_frame + 1) && _queue.front().eyes == EYES_LEFT) {
 206                 return true;
 207         }
 208
 209         return false;
 210 }
 211
 212 void
 213 Writer::thread ()
 214 try
 215 {
 216         while (1)
 217         {
 218                 boost::mutex::scoped_lock lock (_mutex);
 219
 220                 while (1) {
 221
 222                         if (_finish || _queued_full_in_memory > _maximum_frames_in_memory || have_sequenced_image_at_queue_head ()) {
 223                                 break;
 224                         }
 225
 226                         TIMING (N_("writer sleeps with a queue of %1"), _queue.size());
 227                         _condition.wait (lock);
 228                         TIMING (N_("writer wakes with a queue of %1"), _queue.size());
 229                 }
 230
 231                 if (_finish && _queue.empty()) {
 232                         return;
 233                 }
 234
 235                 /* Write any frames that we can write; i.e. those that are in sequence. */
 236                 while (have_sequenced_image_at_queue_head ()) {
 237                         QueueItem qi = _queue.front ();
 238                         _queue.pop_front ();
 239                         if (qi.type == QueueItem::FULL && qi.encoded) {
 240                                 --_queued_full_in_memory;
 241                         }
 242
 243                         lock.unlock ();
 244                         switch (qi.type) {
 245                         case QueueItem::FULL:
 246                         {
 247                                 _film->log()->log (String::compose (N_("Writer FULL-writes %1 to MXF"), qi.frame));
 248                                 if (!qi.encoded) {
 249                                         qi.encoded.reset (new EncodedData (_film->j2c_path (qi.frame, qi.eyes, false)));
 250                                 }
 251
 252                                 libdcp::FrameInfo fin = _picture_asset_writer->write (qi.encoded->data(), qi.encoded->size());
 253                                 qi.encoded->write_info (_film, qi.frame, qi.eyes, fin);
 254                                 _last_written[qi.eyes] = qi.encoded;
 255                                 ++_full_written;
 256                                 break;
 257                         }
 258                         case QueueItem::FAKE:
 259                                 _film->log()->log (String::compose (N_("Writer FAKE-writes %1 to MXF"), qi.frame));
 260                                 _picture_asset_writer->fake_write (qi.size);
 261                                 _last_written[qi.eyes].reset ();
 262                                 ++_fake_written;
 263                                 break;
 264                         case QueueItem::REPEAT:
 265                         {
 266                                 _film->log()->log (String::compose (N_("Writer REPEAT-writes %1 to MXF"), qi.frame));
 267                                 libdcp::FrameInfo fin = _picture_asset_writer->write (
 268                                         _last_written[qi.eyes]->data(),
 269                                         _last_written[qi.eyes]->size()
 270                                         );
 271
 272                                 _last_written[qi.eyes]->write_info (_film, qi.frame, qi.eyes, fin);
 273                                 ++_repeat_written;
 274                                 break;
 275                         }
 276                         }
 277                         lock.lock ();
 278
 279                         _last_written_frame = qi.frame;
 280                         _last_written_eyes = qi.eyes;
 281
 282                         if (_film->length()) {
 283                                 shared_ptr<Job> job = _job.lock ();
 284                                 assert (job);
 285                                 int total = _film->time_to_video_frames (_film->length ());
 286                                 if (_film->three_d ()) {
 287                                         /* _full_written and so on are incremented for each eye, so we need to double the total
 288                                            frames to get the correct progress.
 289                                         */
 290                                         total *= 2;
 291                                 }
 292                                 job->set_progress (float (_full_written + _fake_written + _repeat_written) / total);
 293                         }
 294                 }
 295
 296                 while (_queued_full_in_memory > _maximum_frames_in_memory) {
 297                         /* Too many frames in memory which can't yet be written to the stream.
 298                            Write some FULL frames to disk.
 299                         */
 300
 301                         /* Find one from the back of the queue */
 302                         _queue.sort ();
 303                         list<QueueItem>::reverse_iterator i = _queue.rbegin ();
 304                         while (i != _queue.rend() && (i->type != QueueItem::FULL || !i->encoded)) {
 305                                 ++i;
 306                         }
 307
 308                         assert (i != _queue.rend());
 309                         QueueItem qi = *i;
 310
 311                         ++_pushed_to_disk;
 312
 313                         lock.unlock ();
 314
 315                         _film->log()->log (
 316                                 String::compose (
 317                                         "Writer full (awaiting %1 [last eye was %2]); pushes %3 to disk",
 318                                         _last_written_frame + 1,
 319                                         _last_written_eyes, qi.frame)
 320                                 );
 321
 322                         qi.encoded->write (_film, qi.frame, qi.eyes);
 323                         lock.lock ();
 324                         qi.encoded.reset ();
 325                         --_queued_full_in_memory;
 326                 }
 327         }
 328 }
 329 catch (...)
 330 {
 331         store_current ();
 332 }
 333
 334 void
 335 Writer::finish ()
 336 {
 337         if (!_thread) {
 338                 return;
 339         }
 340
 341         boost::mutex::scoped_lock lock (_mutex);
 342         _finish = true;
 343         _condition.notify_all ();
 344         lock.unlock ();
 345
 346         _thread->join ();
 347         rethrow ();
 348
 349         delete _thread;
 350         _thread = 0;
 351
 352         _picture_asset_writer->finalize ();
 353         _sound_asset_writer->finalize ();
 354
 355         int const frames = _last_written_frame + 1;
 356
 357         _picture_asset->set_duration (frames);
 358
 359         /* Hard-link the video MXF into the DCP */
 360         boost::filesystem::path video_from;
 361         video_from /= _film->internal_video_mxf_dir();
 362         video_from /= _film->internal_video_mxf_filename();
 363
 364         boost::filesystem::path video_to;
 365         video_to /= _film->dir (_film->dcp_name());
 366         video_to /= _film->video_mxf_filename ();
 367
 368         boost::system::error_code ec;
 369         boost::filesystem::create_hard_link (video_from, video_to, ec);
 370         if (ec) {
 371                 /* hard link failed; copy instead */
 372                 boost::filesystem::copy_file (video_from, video_to);
 373                 _film->log()->log ("Hard-link failed; fell back to copying");
 374         }
 375
 376         /* And update the asset */
 377
 378         _picture_asset->set_directory (_film->dir (_film->dcp_name ()));
 379         _picture_asset->set_file_name (_film->video_mxf_filename ());
 380
 381         /* Move the audio MXF into the DCP */
 382
 383         boost::filesystem::path audio_to;
 384         audio_to /= _film->dir (_film->dcp_name ());
 385         audio_to /= _film->audio_mxf_filename ();
 386
 387         boost::filesystem::rename (_film->file (_film->audio_mxf_filename ()), audio_to, ec);
 388         if (ec) {
 389                 throw FileError (
 390                         String::compose (_("could not move audio MXF into the DCP (%1)"), ec.value ()), _film->file (_film->audio_mxf_filename ())
 391                         );
 392         }
 393
 394         _sound_asset->set_directory (_film->dir (_film->dcp_name ()));
 395         _sound_asset->set_duration (frames);
 396
 397         libdcp::DCP dcp (_film->dir (_film->dcp_name()));
 398
 399         shared_ptr<libdcp::CPL> cpl (
 400                 new libdcp::CPL (
 401                         _film->dir (_film->dcp_name()),
 402                         _film->dcp_name(),
 403                         _film->dcp_content_type()->libdcp_kind (),
 404                         frames,
 405                         _film->video_frame_rate ()
 406                         )
 407                 );
 408
 409         dcp.add_cpl (cpl);
 410
 411         cpl->add_reel (shared_ptr<libdcp::Reel> (new libdcp::Reel (
 412                                                          _picture_asset,
 413                                                          _sound_asset,
 414                                                          shared_ptr<libdcp::SubtitleAsset> ()
 415                                                          )
 416                                ));
 417
 418         shared_ptr<Job> job = _job.lock ();
 419         assert (job);
 420
 421         job->sub (_("Computing image digest"));
 422         _picture_asset->compute_digest (boost::bind (&Job::set_progress, job.get(), _1, false));
 423
 424         job->sub (_("Computing audio digest"));
 425         _sound_asset->compute_digest (boost::bind (&Job::set_progress, job.get(), _1, false));
 426
 427         libdcp::XMLMetadata meta = Config::instance()->dcp_metadata ();
 428         meta.set_issue_date_now ();
 429         dcp.write_xml (_film->interop (), meta, _film->is_signed() ? make_signer () : shared_ptr<const libdcp::Signer> ());
 430
 431         _film->log()->log (String::compose (N_("Wrote %1 FULL, %2 FAKE, %3 REPEAT; %4 pushed to disk"), _full_written, _fake_written, _repeat_written, _pushed_to_disk));
 432 }
 433
 434 /** Tell the writer that frame `f' should be a repeat of the frame before it */
 435 void
 436 Writer::repeat (int f, Eyes e)
 437 {
 438         boost::mutex::scoped_lock lock (_mutex);
 439
 440         QueueItem qi;
 441         qi.type = QueueItem::REPEAT;
 442         qi.frame = f;
 443         if (_film->three_d() && e == EYES_BOTH) {
 444                 qi.eyes = EYES_LEFT;
 445                 _queue.push_back (qi);
 446                 qi.eyes = EYES_RIGHT;
 447                 _queue.push_back (qi);
 448         } else {
 449                 qi.eyes = e;
 450                 _queue.push_back (qi);
 451         }
 452
 453         _condition.notify_all ();
 454 }
 455
 456 bool
 457 Writer::check_existing_picture_mxf_frame (FILE* mxf, int f, Eyes eyes)
 458 {
 459         /* Read the frame info as written */
 460         FILE* ifi = fopen_boost (_film->info_path (f, eyes), "r");
 461         if (!ifi) {
 462                 _film->log()->log (String::compose ("Existing frame %1 has no info file", f));
 463                 return false;
 464         }
 465
 466         libdcp::FrameInfo info (ifi);
 467         fclose (ifi);
 468         if (info.size == 0) {
 469                 _film->log()->log (String::compose ("Existing frame %1 has no info file", f));
 470                 return false;
 471         }
 472
 473         /* Read the data from the MXF and hash it */
 474         dcpomatic_fseek (mxf, info.offset, SEEK_SET);
 475         EncodedData data (info.size);
 476         size_t const read = fread (data.data(), 1, data.size(), mxf);
 477         if (read != static_cast<size_t> (data.size ())) {
 478                 _film->log()->log (String::compose ("Existing frame %1 is incomplete", f));
 479                 return false;
 480         }
 481
 482         string const existing_hash = md5_digest (data.data(), data.size());
 483         if (existing_hash != info.hash) {
 484                 _film->log()->log (String::compose ("Existing frame %1 failed hash check", f));
 485                 return false;
 486         }
 487
 488         return true;
 489 }
 490
 491 void
 492 Writer::check_existing_picture_mxf ()
 493 {
 494         /* Try to open the existing MXF */
 495         boost::filesystem::path p;
 496         p /= _film->internal_video_mxf_dir ();
 497         p /= _film->internal_video_mxf_filename ();
 498         FILE* mxf = fopen_boost (p, "rb");
 499         if (!mxf) {
 500                 _film->log()->log (String::compose ("Could not open existing MXF at %1 (errno=%2)", p.string(), errno));
 501                 return;
 502         }
 503
 504         int N = 0;
 505         for (boost::filesystem::directory_iterator i (_film->info_dir ()); i != boost::filesystem::directory_iterator (); ++i) {
 506                 ++N;
 507         }
 508
 509         while (1) {
 510
 511                 shared_ptr<Job> job = _job.lock ();
 512                 assert (job);
 513
 514                 job->set_progress (float (_first_nonexistant_frame) / N);
 515
 516                 if (_film->three_d ()) {
 517                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_LEFT)) {
 518                                 break;
 519                         }
 520                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_RIGHT)) {
 521                                 break;
 522                         }
 523                 } else {
 524                         if (!check_existing_picture_mxf_frame (mxf, _first_nonexistant_frame, EYES_BOTH)) {
 525                                 break;
 526                         }
 527                 }
 528
 529                 _film->log()->log (String::compose ("Have existing frame %1", _first_nonexistant_frame));
 530                 ++_first_nonexistant_frame;
 531         }
 532
 533         fclose (mxf);
 534 }
 535
 536 /** @param frame Frame index.
 537  *  @return true if we can fake-write this frame.
 538  */
 539 bool
 540 Writer::can_fake_write (int frame) const
 541 {
 542         /* We have to do a proper write of the first frame so that we can set up the JPEG2000
 543            parameters in the MXF writer.
 544         */
 545         return (frame != 0 && frame < _first_nonexistant_frame);
 546 }
 547
 548 bool
 549 operator< (QueueItem const & a, QueueItem const & b)
 550 {
 551         if (a.frame != b.frame) {
 552                 return a.frame < b.frame;
 553         }
 554
 555         return static_cast<int> (a.eyes) < static_cast<int> (b.eyes);
 556 }
 557
 558 bool
 559 operator== (QueueItem const & a, QueueItem const & b)
 560 {
 561         return a.frame == b.frame && a.eyes == b.eyes;
 562 }