/*
- Copyright (C) 2018-2019 Carl Hetherington <cth@carlh.net>
+ Copyright (C) 2018-2020 Carl Hetherington <cth@carlh.net>
This file is part of libdcp.
#include "reel.h"
#include "reel_picture_asset.h"
#include "reel_sound_asset.h"
+#include "reel_subtitle_asset.h"
+#include "interop_subtitle_asset.h"
+#include "mono_picture_asset.h"
+#include "mono_picture_frame.h"
+#include "stereo_picture_asset.h"
+#include "stereo_picture_frame.h"
#include "exceptions.h"
#include "compose.hpp"
#include "raw_convert.h"
#include <xercesc/dom/DOMAttr.hpp>
#include <xercesc/dom/DOMErrorHandler.hpp>
#include <xercesc/framework/LocalFileInputSource.hpp>
+#include <xercesc/framework/MemBufInputSource.hpp>
#include <boost/noncopyable.hpp>
#include <boost/foreach.hpp>
#include <boost/algorithm/string.hpp>
using std::string;
using std::cout;
using std::map;
+using std::max;
using boost::shared_ptr;
using boost::optional;
using boost::function;
+using boost::dynamic_pointer_cast;
using namespace dcp;
using namespace xercesc;
-enum Result {
- RESULT_GOOD,
- RESULT_CPL_PKL_DIFFER,
- RESULT_BAD
-};
-
static
string
xml_ch_to_string (XMLCh const * a)
add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd");
add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd");
add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd");
+ add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd");
+ add("interop-subs", "DCSubtitle.v1.mattsson.xsd");
+ add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd");
+ add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd");
+ add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd");
}
InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id)
{
+ if (!system_id) {
+ return 0;
+ }
string system_id_str = xml_ch_to_string (system_id);
+ boost::filesystem::path p = _xsd_dtd_directory;
if (_files.find(system_id_str) == _files.end()) {
- return 0;
+ p /= system_id_str;
+ } else {
+ p /= _files[system_id_str];
}
-
- boost::filesystem::path p = _xsd_dtd_directory / _files[system_id_str];
StringToXMLCh ch (p.string());
return new LocalFileInputSource(ch.get());
}
boost::filesystem::path _xsd_dtd_directory;
};
-static
+
+static void
+parse (XercesDOMParser& parser, boost::filesystem::path xml)
+{
+ parser.parse(xml.string().c_str());
+}
+
+
+static void
+parse (XercesDOMParser& parser, std::string xml)
+{
+ xercesc::MemBufInputSource buf(reinterpret_cast<unsigned char const*>(xml.c_str()), xml.size(), "");
+ parser.parse(buf);
+}
+
+
+template <class T>
void
-validate_xml (boost::filesystem::path xml_file, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
+validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, list<VerificationNote>& notes)
{
try {
XMLPlatformUtils::Initialize ();
parser.setDoNamespaces(true);
parser.setDoSchema(true);
- map<string, string> schema;
- schema["http://www.w3.org/2000/09/xmldsig#"] = "xmldsig-core-schema.xsd";
- schema["http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd"] = "xmldsig-core-schema.xsd";
- schema["http://www.smpte-ra.org/schemas/429-7/2006/CPL"] = "SMPTE-429-7-2006-CPL.xsd";
- schema["http://www.smpte-ra.org/schemas/429-8/2006/PKL"] = "SMPTE-429-8-2006-PKL.xsd";
- schema["http://www.smpte-ra.org/schemas/429-9/2007/AM"] = "SMPTE-429-9-2007-AM.xsd";
- schema["http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd"] = "Main-Stereo-Picture-CPL.xsd";
- schema["http://www.digicine.com/PROTO-ASDCP-CPL-20040511#"] = "PROTO-ASDCP-CPL-20040511.xsd";
- schema["http://www.digicine.com/PROTO-ASDCP-PKL-20040311#"] = "PROTO-ASDCP-PKL-20040311.xsd";
- schema["http://www.digicine.com/PROTO-ASDCP-AM-20040311#"] = "PROTO-ASDCP-AM-20040311.xsd";
-
+ vector<string> schema;
+ schema.push_back("xmldsig-core-schema.xsd");
+ schema.push_back("SMPTE-429-7-2006-CPL.xsd");
+ schema.push_back("SMPTE-429-8-2006-PKL.xsd");
+ schema.push_back("SMPTE-429-9-2007-AM.xsd");
+ schema.push_back("Main-Stereo-Picture-CPL.xsd");
+ schema.push_back("PROTO-ASDCP-CPL-20040511.xsd");
+ schema.push_back("PROTO-ASDCP-PKL-20040311.xsd");
+ schema.push_back("PROTO-ASDCP-AM-20040311.xsd");
+ schema.push_back("DCSubtitle.v1.mattsson.xsd");
+ schema.push_back("DCDMSubtitle-2010.xsd");
+ schema.push_back("PROTO-ASDCP-CC-CPL-20070926.xsd");
+ schema.push_back("SMPTE-429-16.xsd");
+ schema.push_back("Dolby-2012-AD.xsd");
+
+ /* XXX: I'm not especially clear what this is for, but it seems to be necessary */
string locations;
- for (map<string, string>::const_iterator i = schema.begin(); i != schema.end(); ++i) {
- locations += i->first;
- locations += " ";
- boost::filesystem::path p = xsd_dtd_directory / i->second;
- locations += p.string() + " ";
+ BOOST_FOREACH (string i, schema) {
+ locations += String::compose("%1 %1 ", i, i);
}
parser.setExternalSchemaLocation(locations.c_str());
try {
parser.resetDocumentPool();
- parser.parse(xml_file.string().c_str());
+ parse(parser, xml);
} catch (XMLException& e) {
throw MiscError(xml_ch_to_string(e.getMessage()));
} catch (DOMException& e) {
VerificationNote::VERIFY_ERROR,
VerificationNote::XML_VALIDATION_ERROR,
i.message(),
- xml_file,
+ xml,
i.line()
)
);
}
}
-static Result
-verify_asset (shared_ptr<DCP> dcp, shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
+
+enum VerifyAssetResult {
+ VERIFY_ASSET_RESULT_GOOD,
+ VERIFY_ASSET_RESULT_CPL_PKL_DIFFER,
+ VERIFY_ASSET_RESULT_BAD
+};
+
+
+static VerifyAssetResult
+verify_asset (shared_ptr<const DCP> dcp, shared_ptr<const ReelMXF> reel_mxf, function<void (float)> progress)
{
string const actual_hash = reel_mxf->asset_ref()->hash(progress);
optional<string> cpl_hash = reel_mxf->hash();
if (cpl_hash && *cpl_hash != *pkl_hash) {
- return RESULT_CPL_PKL_DIFFER;
+ return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER;
}
if (actual_hash != *pkl_hash) {
- return RESULT_BAD;
+ return VERIFY_ASSET_RESULT_BAD;
+ }
+
+ return VERIFY_ASSET_RESULT_GOOD;
+}
+
+
+enum VerifyPictureAssetResult
+{
+ VERIFY_PICTURE_ASSET_RESULT_GOOD,
+ VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG,
+ VERIFY_PICTURE_ASSET_RESULT_BAD,
+};
+
+
+int
+biggest_frame_size (shared_ptr<const MonoPictureFrame> frame)
+{
+ return frame->j2k_size ();
+}
+
+int
+biggest_frame_size (shared_ptr<const StereoPictureFrame> frame)
+{
+ return max(frame->left_j2k_size(), frame->right_j2k_size());
+}
+
+
+template <class A, class R, class F>
+optional<VerifyPictureAssetResult>
+verify_picture_asset_type (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
+{
+ shared_ptr<A> asset = dynamic_pointer_cast<A>(reel_mxf->asset_ref().asset());
+ if (!asset) {
+ return optional<VerifyPictureAssetResult>();
+ }
+
+ int biggest_frame = 0;
+ shared_ptr<R> reader = asset->start_read ();
+ int64_t const duration = asset->intrinsic_duration ();
+ for (int64_t i = 0; i < duration; ++i) {
+ shared_ptr<const F> frame = reader->get_frame (i);
+ biggest_frame = max(biggest_frame, biggest_frame_size(frame));
+ progress (float(i) / duration);
+ }
+
+ static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float()));
+ static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float()));
+ if (biggest_frame > max_frame) {
+ return VERIFY_PICTURE_ASSET_RESULT_BAD;
+ } else if (biggest_frame > risky_frame) {
+ return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG;
+ }
+
+ return VERIFY_PICTURE_ASSET_RESULT_GOOD;
+}
+
+
+static VerifyPictureAssetResult
+verify_picture_asset (shared_ptr<ReelMXF> reel_mxf, function<void (float)> progress)
+{
+ optional<VerifyPictureAssetResult> r = verify_picture_asset_type<MonoPictureAsset, MonoPictureAssetReader, MonoPictureFrame>(reel_mxf, progress);
+ if (!r) {
+ r = verify_picture_asset_type<StereoPictureAsset, StereoPictureAssetReader, StereoPictureFrame>(reel_mxf, progress);
+ }
+
+ DCP_ASSERT (r);
+ return *r;
+}
+
+
+static void
+verify_main_picture_asset (
+ shared_ptr<const DCP> dcp,
+ shared_ptr<const Reel> reel,
+ function<void (string, optional<boost::filesystem::path>)> stage,
+ function<void (float)> progress,
+ list<VerificationNote>& notes
+ )
+{
+ boost::filesystem::path const file = *reel->main_picture()->asset()->file();
+ stage ("Checking picture asset hash", file);
+ VerifyAssetResult const r = verify_asset (dcp, reel->main_picture(), progress);
+ switch (r) {
+ case VERIFY_ASSET_RESULT_BAD:
+ notes.push_back (
+ VerificationNote(
+ VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file
+ )
+ );
+ break;
+ case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
+ notes.push_back (
+ VerificationNote(
+ VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, file
+ )
+ );
+ break;
+ default:
+ break;
+ }
+ stage ("Checking picture frame sizes", reel->main_picture()->asset()->file());
+ VerifyPictureAssetResult const pr = verify_picture_asset (reel->main_picture(), progress);
+ switch (pr) {
+ case VERIFY_PICTURE_ASSET_RESULT_BAD:
+ notes.push_back (
+ VerificationNote(
+ VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE, file
+ )
+ );
+ break;
+ case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_BIG:
+ notes.push_back (
+ VerificationNote(
+ VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE, file
+ )
+ );
+ break;
+ default:
+ break;
+ }
+}
+
+
+static void
+verify_main_sound_asset (
+ shared_ptr<const DCP> dcp,
+ shared_ptr<const Reel> reel,
+ function<void (string, optional<boost::filesystem::path>)> stage,
+ function<void (float)> progress,
+ list<VerificationNote>& notes
+ )
+{
+ stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
+ VerifyAssetResult const r = verify_asset (dcp, reel->main_sound(), progress);
+ switch (r) {
+ case VERIFY_ASSET_RESULT_BAD:
+ notes.push_back (
+ VerificationNote(
+ VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
+ )
+ );
+ break;
+ case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER:
+ notes.push_back (
+ VerificationNote(
+ VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
+ )
+ );
+ break;
+ default:
+ break;
}
+}
+
- return RESULT_GOOD;
+static void
+verify_main_subtitle_asset (
+ shared_ptr<const Reel> reel,
+ function<void (string, optional<boost::filesystem::path>)> stage,
+ boost::filesystem::path xsd_dtd_directory,
+ list<VerificationNote>& notes
+ )
+{
+ shared_ptr<ReelSubtitleAsset> reel_asset = reel->main_subtitle ();
+ stage ("Checking subtitle XML", reel->main_subtitle()->asset()->file());
+ /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk
+ * gets passed through libdcp which may clean up and therefore hide errors.
+ */
+ validate_xml (reel->main_subtitle()->asset()->raw_xml(), xsd_dtd_directory, notes);
}
stage ("Checking DCP", dcp->directory());
try {
dcp->read (¬es);
- } catch (DCPReadError& e) {
+ } catch (ReadError& e) {
notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
} catch (XMLError& e) {
notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
+ } catch (cxml::Error& e) {
+ notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what())));
}
BOOST_FOREACH (shared_ptr<CPL> cpl, dcp->cpls()) {
BOOST_FOREACH (shared_ptr<Reel> reel, cpl->reels()) {
stage ("Checking reel", optional<boost::filesystem::path>());
+
+ BOOST_FOREACH (shared_ptr<ReelAsset> i, reel->assets()) {
+ if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
+ notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id()));
+ }
+ if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) {
+ notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id()));
+ }
+ }
+
if (reel->main_picture()) {
/* Check reel stuff */
Fraction const frame_rate = reel->main_picture()->frame_rate();
}
/* Check asset */
if (reel->main_picture()->asset_ref().resolved()) {
- stage ("Checking picture asset hash", reel->main_picture()->asset()->file());
- Result const r = verify_asset (dcp, reel->main_picture(), progress);
- switch (r) {
- case RESULT_BAD:
- notes.push_back (
- VerificationNote(
- VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, *reel->main_picture()->asset()->file()
- )
- );
- break;
- case RESULT_CPL_PKL_DIFFER:
- notes.push_back (
- VerificationNote(
- VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE, *reel->main_picture()->asset()->file()
- )
- );
- break;
- default:
- break;
- }
+ verify_main_picture_asset (dcp, reel, stage, progress, notes);
}
}
+
if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
- stage ("Checking sound asset hash", reel->main_sound()->asset()->file());
- Result const r = verify_asset (dcp, reel->main_sound(), progress);
- switch (r) {
- case RESULT_BAD:
- notes.push_back (
- VerificationNote(
- VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *reel->main_sound()->asset()->file()
- )
- );
- break;
- case RESULT_CPL_PKL_DIFFER:
- notes.push_back (
- VerificationNote(
- VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE, *reel->main_sound()->asset()->file()
- )
- );
- break;
- default:
- break;
- }
+ verify_main_sound_asset (dcp, reel, stage, progress, notes);
+ }
+
+ if (reel->main_subtitle() && reel->main_subtitle()->asset_ref().resolved()) {
+ verify_main_subtitle_asset (reel, stage, xsd_dtd_directory, notes);
}
}
}
validate_xml (pkl->file().get(), xsd_dtd_directory, notes);
}
- stage ("Checking ASSETMAP", dcp->asset_map_path().get());
- validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
-
+ if (dcp->asset_map_path()) {
+ stage ("Checking ASSETMAP", dcp->asset_map_path().get());
+ validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes);
+ } else {
+ notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP));
+ }
}
return notes;
case dcp::VerificationNote::GENERAL_READ:
return *note.note();
case dcp::VerificationNote::CPL_HASH_INCORRECT:
- return "The hash of the CPL in the PKL does not agree with the CPL file";
+ return "The hash of the CPL in the PKL does not agree with the CPL file.";
case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE:
- return "The picture in a reel has an invalid frame rate";
+ return "The picture in a reel has an invalid frame rate.";
case dcp::VerificationNote::PICTURE_HASH_INCORRECT:
- return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file", note.file()->filename());
+ return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename());
case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DISAGREE:
- return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1", note.file()->filename());
+ return dcp::String::compose("The PKL and CPL hashes disagree for the picture asset %1.", note.file()->filename());
case dcp::VerificationNote::SOUND_HASH_INCORRECT:
- return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file", note.file()->filename());
+ return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename());
case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DISAGREE:
- return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1", note.file()->filename());
+ return dcp::String::compose("The PKL and CPL hashes disagree for the sound asset %1.", note.file()->filename());
case dcp::VerificationNote::EMPTY_ASSET_PATH:
return "The asset map contains an empty asset path.";
case dcp::VerificationNote::MISSING_ASSET:
return "The DCP contains both SMPTE and Interop parts.";
case dcp::VerificationNote::XML_VALIDATION_ERROR:
return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get());
+ case dcp::VerificationNote::MISSING_ASSETMAP:
+ return "No ASSETMAP or ASSETMAP.xml was found.";
+ case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL:
+ return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get());
+ case dcp::VerificationNote::DURATION_TOO_SMALL:
+ return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get());
+ case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE:
+ return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename());
+ case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE:
+ return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename());
+ case dcp::VerificationNote::EXTERNAL_ASSET:
+ return String::compose("An asset that this DCP refers to is not included in the DCP. It may be a VF. Missing asset is %1.", note.note().get());
}
return "";