src/dcp.cc

   1 /*
   2     Copyright (C) 2012-2021 Carl Hetherington <cth@carlh.net>
   3
   4     This file is part of libdcp.
   5
   6     libdcp is free software; you can redistribute it and/or modify
   7     it under the terms of the GNU General Public License as published by
   8     the Free Software Foundation; either version 2 of the License, or
   9     (at your option) any later version.
  10
  11     libdcp is distributed in the hope that it will be useful,
  12     but WITHOUT ANY WARRANTY; without even the implied warranty of
  13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14     GNU General Public License for more details.
  15
  16     You should have received a copy of the GNU General Public License
  17     along with libdcp.  If not, see <http://www.gnu.org/licenses/>.
  18
  19     In addition, as a special exception, the copyright holders give
  20     permission to link the code of portions of this program with the
  21     OpenSSL library under certain conditions as described in each
  22     individual source file, and distribute linked combinations
  23     including the two.
  24
  25     You must obey the GNU General Public License in all respects
  26     for all of the code used other than OpenSSL.  If you modify
  27     file(s) with this exception, you may extend this exception to your
  28     version of the file(s), but you are not obligated to do so.  If you
  29     do not wish to do so, delete this exception statement from your
  30     version.  If you delete this exception statement from all source
  31     files in the program, then also delete it here.
  32 */
  33
  34
  35 /** @file  src/dcp.cc
  36  *  @brief DCP class
  37  */
  38
  39
  40 #include "asset_factory.h"
  41 #include "atmos_asset.h"
  42 #include "certificate_chain.h"
  43 #include "compose.hpp"
  44 #include "cpl.h"
  45 #include "dcp.h"
  46 #include "dcp_assert.h"
  47 #include "decrypted_kdm.h"
  48 #include "decrypted_kdm_key.h"
  49 #include "exceptions.h"
  50 #include "filesystem.h"
  51 #include "font_asset.h"
  52 #include "interop_subtitle_asset.h"
  53 #include "metadata.h"
  54 #include "mono_j2k_picture_asset.h"
  55 #include "mono_mpeg2_picture_asset.h"
  56 #include "j2k_picture_asset.h"
  57 #include "pkl.h"
  58 #include "raw_convert.h"
  59 #include "reel_asset.h"
  60 #include "reel_subtitle_asset.h"
  61 #include "smpte_subtitle_asset.h"
  62 #include "sound_asset.h"
  63 #include "stereo_j2k_picture_asset.h"
  64 #include "util.h"
  65 #include "verify.h"
  66 #include "warnings.h"
  67 LIBDCP_DISABLE_WARNINGS
  68 #include <asdcp/AS_DCP.h>
  69 LIBDCP_ENABLE_WARNINGS
  70 #include <xmlsec/xmldsig.h>
  71 #include <xmlsec/app.h>
  72 LIBDCP_DISABLE_WARNINGS
  73 #include <libxml++/libxml++.h>
  74 LIBDCP_ENABLE_WARNINGS
  75 #include <boost/algorithm/string.hpp>
  76 #include <numeric>
  77
  78
  79 using std::cerr;
  80 using std::cout;
  81 using std::dynamic_pointer_cast;
  82 using std::exception;
  83 using std::list;
  84 using std::make_pair;
  85 using std::make_shared;
  86 using std::map;
  87 using std::shared_ptr;
  88 using std::string;
  89 using std::vector;
  90 using boost::algorithm::starts_with;
  91 using boost::optional;
  92 using namespace dcp;
  93
  94
  95 static string const volindex_interop_ns = "http://www.digicine.com/PROTO-ASDCP-VL-20040311#";
  96 static string const volindex_smpte_ns   = "http://www.smpte-ra.org/schemas/429-9/2007/AM";
  97
  98
  99 DCP::DCP (boost::filesystem::path directory)
 100         : _directory (directory)
 101 {
 102         if (!filesystem::exists(directory)) {
 103                 filesystem::create_directories(directory);
 104         }
 105
 106         _directory = filesystem::canonical(_directory);
 107 }
 108
 109
 110 DCP::DCP(DCP&& other)
 111         : _directory(std::move(other._directory))
 112         , _cpls(std::move(other._cpls))
 113         , _pkls(std::move(other._pkls))
 114         , _asset_map(std::move(other._asset_map))
 115         , _new_issuer(std::move(other._new_issuer))
 116         , _new_creator(std::move(other._new_creator))
 117         , _new_issue_date(std::move(other._new_issue_date))
 118         , _new_annotation_text(std::move(other._new_annotation_text))
 119 {
 120
 121 }
 122
 123
 124 DCP&
 125 DCP::operator=(DCP&& other)
 126 {
 127         _directory = std::move(other._directory);
 128         _cpls = std::move(other._cpls);
 129         _pkls = std::move(other._pkls);
 130         _asset_map = std::move(other._asset_map);
 131         _new_issuer = std::move(other._new_issuer);
 132         _new_creator = std::move(other._new_creator);
 133         _new_issue_date = std::move(other._new_issue_date);
 134         _new_annotation_text = std::move(other._new_annotation_text);
 135         return *this;
 136 }
 137
 138
 139 void
 140 DCP::read (vector<dcp::VerificationNote>* notes, bool ignore_incorrect_picture_mxf_type)
 141 {
 142         /* Read the ASSETMAP and PKL */
 143
 144         boost::filesystem::path asset_map_path;
 145         if (filesystem::exists(_directory / "ASSETMAP")) {
 146                 asset_map_path = _directory / "ASSETMAP";
 147         } else if (filesystem::exists(_directory / "ASSETMAP.xml")) {
 148                 asset_map_path = _directory / "ASSETMAP.xml";
 149         } else {
 150                 boost::throw_exception(MissingAssetmapError(_directory));
 151         }
 152
 153         _asset_map = AssetMap(asset_map_path);
 154         auto const pkl_paths = _asset_map->pkl_paths();
 155         auto const standard = _asset_map->standard();
 156
 157         if (pkl_paths.empty()) {
 158                 boost::throw_exception (XMLError ("No packing lists found in asset map"));
 159         }
 160
 161         for (auto i: pkl_paths) {
 162                 _pkls.push_back(make_shared<PKL>(i));
 163         }
 164
 165         /* Now we have:
 166              paths - map of files in the DCP that are not PKLs; key is ID, value is path.
 167              _pkls - PKL objects for each PKL.
 168
 169            Read all the assets from the asset map.
 170          */
 171
 172         /* Make a list of non-CPL/PKL assets so that we can resolve the references
 173            from the CPLs.
 174         */
 175         vector<shared_ptr<Asset>> other_assets;
 176
 177         auto ids_and_paths = _asset_map->asset_ids_and_paths();
 178         for (auto id_and_path: ids_and_paths) {
 179                 auto const id = id_and_path.first;
 180                 auto const path = id_and_path.second;
 181
 182                 if (path == _directory) {
 183                         /* I can't see how this is valid, but it's
 184                            been seen in the wild with a DCP that
 185                            claims to come from ClipsterDCI 5.10.0.5.
 186                         */
 187                         if (notes) {
 188                                 notes->push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::EMPTY_ASSET_PATH});
 189                         }
 190                         continue;
 191                 }
 192
 193                 if (!filesystem::exists(path)) {
 194                         if (notes) {
 195                                 notes->push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::MISSING_ASSET, path});
 196                         }
 197                         continue;
 198                 }
 199
 200                 /* Find the <Type> for this asset from the PKL that contains the asset */
 201                 optional<string> pkl_type;
 202                 for (auto j: _pkls) {
 203                         pkl_type = j->type(id);
 204                         if (pkl_type) {
 205                                 break;
 206                         }
 207                 }
 208
 209                 if (!pkl_type) {
 210                         /* This asset is in the ASSETMAP but not mentioned in any PKL so we don't
 211                          * need to worry about it.
 212                          */
 213                         continue;
 214                 }
 215
 216                 auto remove_parameters = [](string const& n) {
 217                         return n.substr(0, n.find(";"));
 218                 };
 219
 220                 /* Remove any optional parameters (after ;) */
 221                 pkl_type = pkl_type->substr(0, pkl_type->find(";"));
 222
 223                 if (
 224                         pkl_type == remove_parameters(CPL::static_pkl_type(standard)) ||
 225                         pkl_type == remove_parameters(InteropSubtitleAsset::static_pkl_type(standard))) {
 226                         auto p = new xmlpp::DomParser;
 227                         try {
 228                                 p->parse_file(dcp::filesystem::fix_long_path(path).string());
 229                         } catch (std::exception& e) {
 230                                 delete p;
 231                                 throw ReadError(String::compose("XML error in %1", path.string()), e.what());
 232                         }
 233
 234                         auto const root = p->get_document()->get_root_node()->get_name();
 235                         delete p;
 236
 237                         if (root == "CompositionPlaylist") {
 238                                 auto cpl = make_shared<CPL>(path, notes);
 239                                 if (cpl->standard() != standard && notes) {
 240                                         notes->push_back ({VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_STANDARD});
 241                                 }
 242                                 _cpls.push_back (cpl);
 243                         } else if (root == "DCSubtitle") {
 244                                 if (standard == Standard::SMPTE && notes) {
 245                                         notes->push_back (VerificationNote(VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_STANDARD));
 246                                 }
 247                                 other_assets.push_back (make_shared<InteropSubtitleAsset>(path));
 248                         }
 249                 } else if (
 250                         *pkl_type == remove_parameters(J2KPictureAsset::static_pkl_type(standard)) ||
 251                         *pkl_type == remove_parameters(MPEG2PictureAsset::static_pkl_type(standard)) ||
 252                         *pkl_type == remove_parameters(SoundAsset::static_pkl_type(standard)) ||
 253                         *pkl_type == remove_parameters(AtmosAsset::static_pkl_type(standard)) ||
 254                         *pkl_type == remove_parameters(SMPTESubtitleAsset::static_pkl_type(standard))
 255                         ) {
 256
 257                         bool found_threed_marked_as_twod = false;
 258                         auto asset = asset_factory(path, ignore_incorrect_picture_mxf_type, &found_threed_marked_as_twod);
 259                         if (asset->id() != id) {
 260                                 notes->push_back(VerificationNote(VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_ASSET_MAP_ID).set_id(id).set_other_id(asset->id()));
 261                         }
 262                         other_assets.push_back(asset);
 263                         if (found_threed_marked_as_twod && notes) {
 264                                 notes->push_back ({VerificationNote::Type::WARNING, VerificationNote::Code::THREED_ASSET_MARKED_AS_TWOD, path});
 265                         }
 266                 } else if (*pkl_type == remove_parameters(FontAsset::static_pkl_type(standard))) {
 267                         other_assets.push_back(make_shared<FontAsset>(id, path));
 268                 } else if (*pkl_type == "image/png") {
 269                         /* It's an Interop PNG subtitle; let it go */
 270                 } else {
 271                         throw ReadError (String::compose("Unknown asset type %1 in PKL", *pkl_type));
 272                 }
 273         }
 274
 275         /* Set hashes for assets where we have an idea of what the hash should be in either a CPL or PKL.
 276          * This means that when the hash is later read from these objects the result will be the one that
 277          * it should be, rather the one that it currently is.  This should prevent errors being concealed
 278          * when an asset is corrupted - the hash from the CPL/PKL will disagree with the actual hash of the
 279          * file, revealing the problem.
 280          */
 281
 282         auto hash_from_pkl = [this](string id) -> optional<string> {
 283                 for (auto pkl: _pkls) {
 284                         if (auto pkl_hash = pkl->hash(id)) {
 285                                 return pkl_hash;
 286                         }
 287                 }
 288
 289                 return {};
 290         };
 291
 292         auto hash_from_cpl_or_pkl = [this, &hash_from_pkl](string id) -> optional<string> {
 293                 for (auto cpl: cpls()) {
 294                         for (auto reel_file_asset: cpl->reel_file_assets()) {
 295                                 if (reel_file_asset->asset_ref().id() == id && reel_file_asset->hash()) {
 296                                         return reel_file_asset->hash();
 297                                 }
 298                         }
 299                 }
 300
 301                 return hash_from_pkl(id);
 302         };
 303
 304         for (auto asset: other_assets) {
 305                 if (auto hash = hash_from_cpl_or_pkl(asset->id())) {
 306                         asset->set_hash(*hash);
 307                 }
 308         }
 309
 310         for (auto cpl: cpls()) {
 311                 if (auto hash = hash_from_pkl(cpl->id())) {
 312                         cpl->set_hash(*hash);
 313                 }
 314         }
 315
 316         /* Resolve references */
 317         resolve_refs (other_assets);
 318
 319         /* While we've got the ASSETMAP lets look and see if this DCP refers to things that are not in its ASSETMAP */
 320         if (notes) {
 321                 for (auto i: cpls()) {
 322                         for (auto j: i->reel_file_assets()) {
 323                                 if (!j->asset_ref().resolved() && ids_and_paths.find(j->asset_ref().id()) == ids_and_paths.end()) {
 324                                         notes->push_back (VerificationNote(VerificationNote::Type::WARNING, VerificationNote::Code::EXTERNAL_ASSET, j->asset_ref().id()));
 325                                 }
 326                         }
 327                 }
 328         }
 329 }
 330
 331
 332 void
 333 DCP::resolve_refs (vector<shared_ptr<Asset>> assets)
 334 {
 335         for (auto i: cpls()) {
 336                 i->resolve_refs (assets);
 337         }
 338 }
 339
 340
 341 bool
 342 DCP::equals(DCP const & other, EqualityOptions const& opt, NoteHandler note) const
 343 {
 344         auto a = cpls ();
 345         auto b = other.cpls ();
 346
 347         if (a.size() != b.size()) {
 348                 note (NoteType::ERROR, String::compose ("CPL counts differ: %1 vs %2", a.size(), b.size()));
 349                 return false;
 350         }
 351
 352         bool r = true;
 353
 354         for (auto i: a) {
 355                 auto j = b.begin();
 356                 while (j != b.end() && !(*j)->equals (i, opt, note)) {
 357                         ++j;
 358                 }
 359
 360                 if (j == b.end ()) {
 361                         r = false;
 362                 }
 363         }
 364
 365         return r;
 366 }
 367
 368
 369 void
 370 DCP::add (shared_ptr<CPL> cpl)
 371 {
 372         _cpls.push_back (cpl);
 373 }
 374
 375
 376 bool
 377 DCP::any_encrypted () const
 378 {
 379         for (auto i: cpls()) {
 380                 if (i->any_encrypted()) {
 381                         return true;
 382                 }
 383         }
 384
 385         return false;
 386 }
 387
 388
 389 bool
 390 DCP::all_encrypted () const
 391 {
 392         for (auto i: cpls()) {
 393                 if (!i->all_encrypted()) {
 394                         return false;
 395                 }
 396         }
 397
 398         return true;
 399 }
 400
 401
 402 void
 403 DCP::add (DecryptedKDM const & kdm)
 404 {
 405         auto keys = kdm.keys();
 406         for (auto cpl: cpls()) {
 407                 if (std::any_of(keys.begin(), keys.end(), [cpl](DecryptedKDMKey const& key) { return key.cpl_id() == cpl->id(); })) {
 408                         cpl->add (kdm);
 409                 }
 410         }
 411 }
 412
 413
 414 /** Write the VOLINDEX file.
 415  *  @param standard DCP standard to use (INTEROP or SMPTE)
 416  */
 417 void
 418 DCP::write_volindex (Standard standard) const
 419 {
 420         auto p = _directory;
 421         switch (standard) {
 422         case Standard::INTEROP:
 423                 p /= "VOLINDEX";
 424                 break;
 425         case Standard::SMPTE:
 426                 p /= "VOLINDEX.xml";
 427                 break;
 428         default:
 429                 DCP_ASSERT (false);
 430         }
 431
 432         xmlpp::Document doc;
 433         xmlpp::Element* root;
 434
 435         switch (standard) {
 436         case Standard::INTEROP:
 437                 root = doc.create_root_node ("VolumeIndex", volindex_interop_ns);
 438                 break;
 439         case Standard::SMPTE:
 440                 root = doc.create_root_node ("VolumeIndex", volindex_smpte_ns);
 441                 break;
 442         default:
 443                 DCP_ASSERT (false);
 444         }
 445
 446         cxml::add_text_child(root, "Index", "1");
 447         doc.write_to_file_formatted(dcp::filesystem::fix_long_path(p).string(), "UTF-8");
 448 }
 449
 450
 451 void
 452 DCP::write_xml(shared_ptr<const CertificateChain> signer, bool include_mca_subdescriptors, NameFormat name_format)
 453 {
 454         if (_cpls.empty()) {
 455                 throw MiscError ("Cannot write DCP with no CPLs.");
 456         }
 457
 458         auto standard = std::accumulate (
 459                 std::next(_cpls.begin()), _cpls.end(), _cpls[0]->standard(),
 460                 [](Standard s, shared_ptr<CPL> c) {
 461                         if (s != c->standard()) {
 462                                 throw MiscError ("Cannot make DCP with mixed Interop and SMPTE CPLs.");
 463                         }
 464                         return s;
 465                 }
 466                 );
 467
 468         for (auto i: cpls()) {
 469                 NameFormat::Map values;
 470                 values['t'] = "cpl";
 471                 i->write_xml(_directory / (name_format.get(values, "_" + i->id() + ".xml")), signer, include_mca_subdescriptors);
 472         }
 473
 474         if (_pkls.empty()) {
 475                 _pkls.push_back(
 476                         make_shared<PKL>(
 477                                 standard,
 478                                 _new_annotation_text.get_value_or(String::compose("Created by libdcp %1", dcp::version)),
 479                                 _new_issue_date.get_value_or(LocalTime().as_string()),
 480                                 _new_issuer.get_value_or(String::compose("libdcp %1", dcp::version)),
 481                                 _new_creator.get_value_or(String::compose("libdcp %1", dcp::version))
 482                                 )
 483                         );
 484         }
 485
 486         auto pkl = _pkls.front();
 487
 488         /* The assets may have changed since we read the PKL, so re-add them */
 489         pkl->clear_assets();
 490         for (auto asset: assets()) {
 491                 asset->add_to_pkl(pkl, _directory);
 492         }
 493
 494         NameFormat::Map values;
 495         values['t'] = "pkl";
 496         auto pkl_path = _directory / name_format.get(values, "_" + pkl->id() + ".xml");
 497         pkl->write_xml (pkl_path, signer);
 498
 499         if (!_asset_map) {
 500                 _asset_map = AssetMap(
 501                         standard,
 502                         _new_annotation_text.get_value_or(String::compose("Created by libdcp %1", dcp::version)),
 503                         _new_issue_date.get_value_or(LocalTime().as_string()),
 504                         _new_issuer.get_value_or(String::compose("libdcp %1", dcp::version)),
 505                         _new_creator.get_value_or(String::compose("libdcp %1", dcp::version))
 506                         );
 507         }
 508
 509         /* The assets may have changed since we read the asset map, so re-add them */
 510         _asset_map->clear_assets();
 511         _asset_map->add_asset(pkl->id(), pkl_path, true);
 512         for (auto asset: assets()) {
 513                 asset->add_to_assetmap(*_asset_map, _directory);
 514         }
 515
 516         _asset_map->write_xml(
 517                 _directory / (standard == Standard::INTEROP ? "ASSETMAP" : "ASSETMAP.xml")
 518                 );
 519
 520         write_volindex (standard);
 521 }
 522
 523
 524 vector<shared_ptr<CPL>>
 525 DCP::cpls () const
 526 {
 527         return _cpls;
 528 }
 529
 530
 531 vector<shared_ptr<Asset>>
 532 DCP::assets (bool ignore_unresolved) const
 533 {
 534         vector<shared_ptr<Asset>> assets;
 535         for (auto i: cpls()) {
 536                 assets.push_back (i);
 537                 for (auto j: i->reel_file_assets()) {
 538                         if (ignore_unresolved && !j->asset_ref().resolved()) {
 539                                 continue;
 540                         }
 541
 542                         auto const id = j->asset_ref().id();
 543                         if (std::find_if(assets.begin(), assets.end(), [id](shared_ptr<Asset> asset) { return asset->id() == id; }) == assets.end()) {
 544                                 auto o = j->asset_ref().asset();
 545                                 assets.push_back (o);
 546                                 /* More Interop special-casing */
 547                                 auto sub = dynamic_pointer_cast<InteropSubtitleAsset>(o);
 548                                 if (sub) {
 549                                         add_to_container(assets, sub->font_assets());
 550                                 }
 551                         }
 552                 }
 553         }
 554
 555         return assets;
 556 }
 557
 558
 559 /** Given a list of files that make up 1 or more DCPs, return the DCP directories */
 560 vector<boost::filesystem::path>
 561 DCP::directories_from_files (vector<boost::filesystem::path> files)
 562 {
 563         vector<boost::filesystem::path> d;
 564         for (auto i: files) {
 565                 if (i.filename() == "ASSETMAP" || i.filename() == "ASSETMAP.xml") {
 566                         d.push_back (i.parent_path ());
 567                 }
 568         }
 569         return d;
 570 }
 571
 572
 573 void
 574 DCP::set_issuer(string issuer)
 575 {
 576         for (auto pkl: _pkls) {
 577                 pkl->set_issuer(issuer);
 578         }
 579         if (_asset_map) {
 580                 _asset_map->set_issuer(issuer);
 581         }
 582         _new_issuer = issuer;
 583 }
 584
 585
 586 void
 587 DCP::set_creator(string creator)
 588 {
 589         for (auto pkl: _pkls) {
 590                 pkl->set_creator(creator);
 591         }
 592         if (_asset_map) {
 593                 _asset_map->set_creator(creator);
 594         }
 595         _new_creator = creator;
 596 }
 597
 598
 599 void
 600 DCP::set_issue_date(string issue_date)
 601 {
 602         for (auto pkl: _pkls) {
 603                 pkl->set_issue_date(issue_date);
 604         }
 605         if (_asset_map) {
 606                 _asset_map->set_issue_date(issue_date);
 607         }
 608         _new_issue_date = issue_date;
 609 }
 610
 611
 612 void
 613 DCP::set_annotation_text(string annotation_text)
 614 {
 615         for (auto pkl: _pkls) {
 616                 pkl->set_annotation_text(annotation_text);
 617         }
 618         if (_asset_map) {
 619                 _asset_map->set_annotation_text(annotation_text);
 620         }
 621         _new_annotation_text = annotation_text;
 622 }
 623