/* Copyright (C) 2018-2021 Carl Hetherington This file is part of libdcp. libdcp is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. libdcp is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with libdcp. If not, see . In addition, as a special exception, the copyright holders give permission to link the code of portions of this program with the OpenSSL library under certain conditions as described in each individual source file, and distribute linked combinations including the two. You must obey the GNU General Public License in all respects for all of the code used other than OpenSSL. If you modify file(s) with this exception, you may extend this exception to your version of the file(s), but you are not obligated to do so. If you do not wish to do so, delete this exception statement from your version. If you delete this exception statement from all source files in the program, then also delete it here. */ #include "verify.h" #include "dcp.h" #include "cpl.h" #include "reel.h" #include "reel_closed_caption_asset.h" #include "reel_picture_asset.h" #include "reel_sound_asset.h" #include "reel_subtitle_asset.h" #include "interop_subtitle_asset.h" #include "mono_picture_asset.h" #include "mono_picture_frame.h" #include "stereo_picture_asset.h" #include "stereo_picture_frame.h" #include "exceptions.h" #include "compose.hpp" #include "raw_convert.h" #include "smpte_subtitle_asset.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using std::list; using std::vector; using std::string; using std::cout; using std::map; using std::max; using std::shared_ptr; using std::make_shared; using boost::optional; using boost::function; using std::dynamic_pointer_cast; using namespace dcp; using namespace xercesc; static string xml_ch_to_string (XMLCh const * a) { char* x = XMLString::transcode(a); string const o(x); XMLString::release(&x); return o; } class XMLValidationError { public: XMLValidationError (SAXParseException const & e) : _message (xml_ch_to_string(e.getMessage())) , _line (e.getLineNumber()) , _column (e.getColumnNumber()) , _public_id (e.getPublicId() ? xml_ch_to_string(e.getPublicId()) : "") , _system_id (e.getSystemId() ? xml_ch_to_string(e.getSystemId()) : "") { } string message () const { return _message; } uint64_t line () const { return _line; } uint64_t column () const { return _column; } string public_id () const { return _public_id; } string system_id () const { return _system_id; } private: string _message; uint64_t _line; uint64_t _column; string _public_id; string _system_id; }; class DCPErrorHandler : public ErrorHandler { public: void warning(const SAXParseException& e) { maybe_add (XMLValidationError(e)); } void error(const SAXParseException& e) { maybe_add (XMLValidationError(e)); } void fatalError(const SAXParseException& e) { maybe_add (XMLValidationError(e)); } void resetErrors() { _errors.clear (); } list errors () const { return _errors; } private: void maybe_add (XMLValidationError e) { /* XXX: nasty hack */ if ( e.message().find("schema document") != string::npos && e.message().find("has different target namespace from the one specified in instance document") != string::npos ) { return; } _errors.push_back (e); } list _errors; }; class StringToXMLCh : public boost::noncopyable { public: StringToXMLCh (string a) { _buffer = XMLString::transcode(a.c_str()); } ~StringToXMLCh () { XMLString::release (&_buffer); } XMLCh const * get () const { return _buffer; } private: XMLCh* _buffer; }; class LocalFileResolver : public EntityResolver { public: LocalFileResolver (boost::filesystem::path xsd_dtd_directory) : _xsd_dtd_directory (xsd_dtd_directory) { /* XXX: I'm not clear on what things need to be in this list; some XSDs are apparently, magically * found without being here. */ add("http://www.w3.org/2001/XMLSchema.dtd", "XMLSchema.dtd"); add("http://www.w3.org/2001/03/xml.xsd", "xml.xsd"); add("http://www.w3.org/TR/2002/REC-xmldsig-core-20020212/xmldsig-core-schema.xsd", "xmldsig-core-schema.xsd"); add("http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL.xsd", "Main-Stereo-Picture-CPL.xsd"); add("http://www.digicine.com/PROTO-ASDCP-CPL-20040511.xsd", "PROTO-ASDCP-CPL-20040511.xsd"); add("http://www.digicine.com/PROTO-ASDCP-PKL-20040311.xsd", "PROTO-ASDCP-PKL-20040311.xsd"); add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd"); add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd"); add("interop-subs", "DCSubtitle.v1.mattsson.xsd"); add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd"); add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd"); add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd"); add("http://www.smpte-ra.org/schemas/429-10/2008/Main-Stereo-Picture-CPL", "SMPTE-429-10-2008.xsd"); } InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id) { if (!system_id) { return 0; } auto system_id_str = xml_ch_to_string (system_id); auto p = _xsd_dtd_directory; if (_files.find(system_id_str) == _files.end()) { p /= system_id_str; } else { p /= _files[system_id_str]; } StringToXMLCh ch (p.string()); return new LocalFileInputSource(ch.get()); } private: void add (string uri, string file) { _files[uri] = file; } std::map _files; boost::filesystem::path _xsd_dtd_directory; }; static void parse (XercesDOMParser& parser, boost::filesystem::path xml) { parser.parse(xml.string().c_str()); } static void parse (XercesDOMParser& parser, string xml) { xercesc::MemBufInputSource buf(reinterpret_cast(xml.c_str()), xml.size(), ""); parser.parse(buf); } template void validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, vector& notes) { try { XMLPlatformUtils::Initialize (); } catch (XMLException& e) { throw MiscError ("Failed to initialise xerces library"); } DCPErrorHandler error_handler; /* All the xerces objects in this scope must be destroyed before XMLPlatformUtils::Terminate() is called */ { XercesDOMParser parser; parser.setValidationScheme(XercesDOMParser::Val_Always); parser.setDoNamespaces(true); parser.setDoSchema(true); vector schema; schema.push_back("xml.xsd"); schema.push_back("xmldsig-core-schema.xsd"); schema.push_back("SMPTE-429-7-2006-CPL.xsd"); schema.push_back("SMPTE-429-8-2006-PKL.xsd"); schema.push_back("SMPTE-429-9-2007-AM.xsd"); schema.push_back("Main-Stereo-Picture-CPL.xsd"); schema.push_back("PROTO-ASDCP-CPL-20040511.xsd"); schema.push_back("PROTO-ASDCP-PKL-20040311.xsd"); schema.push_back("PROTO-ASDCP-AM-20040311.xsd"); schema.push_back("DCSubtitle.v1.mattsson.xsd"); schema.push_back("DCDMSubtitle-2010.xsd"); schema.push_back("PROTO-ASDCP-CC-CPL-20070926.xsd"); schema.push_back("SMPTE-429-16.xsd"); schema.push_back("Dolby-2012-AD.xsd"); schema.push_back("SMPTE-429-10-2008.xsd"); schema.push_back("xlink.xsd"); schema.push_back("SMPTE-335-2012.xsd"); schema.push_back("SMPTE-395-2014-13-1-aaf.xsd"); schema.push_back("isdcf-mca.xsd"); schema.push_back("SMPTE-429-12-2008.xsd"); /* XXX: I'm not especially clear what this is for, but it seems to be necessary. * Schemas that are not mentioned in this list are not read, and the things * they describe are not checked. */ string locations; for (auto i: schema) { locations += String::compose("%1 %1 ", i, i); } parser.setExternalSchemaLocation(locations.c_str()); parser.setValidationSchemaFullChecking(true); parser.setErrorHandler(&error_handler); LocalFileResolver resolver (xsd_dtd_directory); parser.setEntityResolver(&resolver); try { parser.resetDocumentPool(); parse(parser, xml); } catch (XMLException& e) { throw MiscError(xml_ch_to_string(e.getMessage())); } catch (DOMException& e) { throw MiscError(xml_ch_to_string(e.getMessage())); } catch (...) { throw MiscError("Unknown exception from xerces"); } } XMLPlatformUtils::Terminate (); for (auto i: error_handler.errors()) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_ERROR, VerificationNote::XML_VALIDATION_ERROR, i.message(), boost::trim_copy(i.public_id() + " " + i.system_id()), i.line() ) ); } } enum VerifyAssetResult { VERIFY_ASSET_RESULT_GOOD, VERIFY_ASSET_RESULT_CPL_PKL_DIFFER, VERIFY_ASSET_RESULT_BAD }; static VerifyAssetResult verify_asset (shared_ptr dcp, shared_ptr reel_mxf, function progress) { auto const actual_hash = reel_mxf->asset_ref()->hash(progress); auto pkls = dcp->pkls(); /* We've read this DCP in so it must have at least one PKL */ DCP_ASSERT (!pkls.empty()); auto asset = reel_mxf->asset_ref().asset(); optional pkl_hash; for (auto i: pkls) { pkl_hash = i->hash (reel_mxf->asset_ref()->id()); if (pkl_hash) { break; } } DCP_ASSERT (pkl_hash); auto cpl_hash = reel_mxf->hash(); if (cpl_hash && *cpl_hash != *pkl_hash) { return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER; } if (actual_hash != *pkl_hash) { return VERIFY_ASSET_RESULT_BAD; } return VERIFY_ASSET_RESULT_GOOD; } void verify_language_tag (string tag, vector& notes) { try { dcp::LanguageTag test (tag); } catch (dcp::LanguageTagError &) { notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::BAD_LANGUAGE, tag)); } } enum VerifyPictureAssetResult { VERIFY_PICTURE_ASSET_RESULT_GOOD, VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE, VERIFY_PICTURE_ASSET_RESULT_BAD, }; int biggest_frame_size (shared_ptr frame) { return frame->size (); } int biggest_frame_size (shared_ptr frame) { return max(frame->left()->size(), frame->right()->size()); } template optional verify_picture_asset_type (shared_ptr reel_mxf, function progress) { auto asset = dynamic_pointer_cast(reel_mxf->asset_ref().asset()); if (!asset) { return optional(); } int biggest_frame = 0; auto reader = asset->start_read (); auto const duration = asset->intrinsic_duration (); for (int64_t i = 0; i < duration; ++i) { shared_ptr frame = reader->get_frame (i); biggest_frame = max(biggest_frame, biggest_frame_size(frame)); progress (float(i) / duration); } static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float())); static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float())); if (biggest_frame > max_frame) { return VERIFY_PICTURE_ASSET_RESULT_BAD; } else if (biggest_frame > risky_frame) { return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE; } return VERIFY_PICTURE_ASSET_RESULT_GOOD; } static VerifyPictureAssetResult verify_picture_asset (shared_ptr reel_mxf, function progress) { auto r = verify_picture_asset_type(reel_mxf, progress); if (!r) { r = verify_picture_asset_type(reel_mxf, progress); } DCP_ASSERT (r); return *r; } static void verify_main_picture_asset ( shared_ptr dcp, shared_ptr reel_asset, function)> stage, function progress, vector& notes ) { auto asset = reel_asset->asset(); auto const file = *asset->file(); stage ("Checking picture asset hash", file); auto const r = verify_asset (dcp, reel_asset, progress); switch (r) { case VERIFY_ASSET_RESULT_BAD: notes.push_back ( VerificationNote( VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_HASH_INCORRECT, file ) ); break; case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER: notes.push_back ( VerificationNote( VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_PICTURE_HASHES_DIFFER, file ) ); break; default: break; } stage ("Checking picture frame sizes", asset->file()); auto const pr = verify_picture_asset (reel_asset, progress); switch (pr) { case VERIFY_PICTURE_ASSET_RESULT_BAD: notes.push_back ( VerificationNote( VerificationNote::VERIFY_ERROR, VerificationNote::PICTURE_FRAME_TOO_LARGE_IN_BYTES, file ) ); break; case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE: notes.push_back ( VerificationNote( VerificationNote::VERIFY_WARNING, VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE_IN_BYTES, file ) ); break; default: break; } /* Only flat/scope allowed by Bv2.1 */ if ( asset->size() != dcp::Size(2048, 858) && asset->size() != dcp::Size(1998, 1080) && asset->size() != dcp::Size(4096, 1716) && asset->size() != dcp::Size(3996, 2160)) { notes.push_back( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::PICTURE_ASSET_INVALID_SIZE_IN_PIXELS, String::compose("%1x%2", asset->size().width, asset->size().height), file ) ); } /* Only 24, 25, 48fps allowed for 2K */ if ( (asset->size() == dcp::Size(2048, 858) || asset->size() == dcp::Size(1998, 1080)) && (asset->edit_rate() != dcp::Fraction(24, 1) && asset->edit_rate() != dcp::Fraction(25, 1) && asset->edit_rate() != dcp::Fraction(48, 1)) ) { notes.push_back( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_2K, String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), file ) ); } if (asset->size() == dcp::Size(4096, 1716) || asset->size() == dcp::Size(3996, 2160)) { /* Only 24fps allowed for 4K */ if (asset->edit_rate() != dcp::Fraction(24, 1)) { notes.push_back( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_4K, String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), file ) ); } /* Only 2D allowed for 4K */ if (dynamic_pointer_cast(asset)) { notes.push_back( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::PICTURE_ASSET_4K_3D, file ) ); } } } static void verify_main_sound_asset ( shared_ptr dcp, shared_ptr reel_asset, function)> stage, function progress, vector& notes ) { auto asset = reel_asset->asset(); stage ("Checking sound asset hash", asset->file()); auto const r = verify_asset (dcp, reel_asset, progress); switch (r) { case VERIFY_ASSET_RESULT_BAD: notes.push_back ( VerificationNote( VerificationNote::VERIFY_ERROR, VerificationNote::SOUND_HASH_INCORRECT, *asset->file() ) ); break; case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER: notes.push_back ( VerificationNote( VerificationNote::VERIFY_ERROR, VerificationNote::PKL_CPL_SOUND_HASHES_DIFFER, *asset->file() ) ); break; default: break; } stage ("Checking sound asset metadata", asset->file()); verify_language_tag (asset->language(), notes); if (asset->sampling_rate() != 48000) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_SOUND_FRAME_RATE, *asset->file() ) ); } } static void verify_main_subtitle_reel (shared_ptr reel_asset, vector& notes) { /* XXX: is Language compulsory? */ if (reel_asset->language()) { verify_language_tag (*reel_asset->language(), notes); } if (!reel_asset->entry_point()) { notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_ENTRY_POINT }); } else if (reel_asset->entry_point().get()) { notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_ENTRY_POINT_NON_ZERO }); } } static void verify_closed_caption_reel (shared_ptr reel_asset, vector& notes) { /* XXX: is Language compulsory? */ if (reel_asset->language()) { verify_language_tag (*reel_asset->language(), notes); } if (!reel_asset->entry_point()) { notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_CLOSED_CAPTION_ENTRY_POINT }); } else if (reel_asset->entry_point().get()) { notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_ENTRY_POINT_NON_ZERO }); } } struct State { boost::optional subtitle_language; }; void verify_smpte_subtitle_asset ( shared_ptr asset, vector& notes, State& state ) { if (asset->language()) { auto const language = *asset->language(); verify_language_tag (language, notes); if (!state.subtitle_language) { state.subtitle_language = language; } else if (state.subtitle_language != language) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_LANGUAGES_DIFFER, *asset->file() ) ); } } else { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_LANGUAGE, *asset->file() ) ); } if (boost::filesystem::file_size(*asset->file()) > 115 * 1024 * 1024) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_ASSET_TOO_LARGE_IN_BYTES, *asset->file() ) ); } /* XXX: I'm not sure what Bv2.1_7.2.1 means when it says "the font resource shall not be larger than 10MB" * but I'm hoping that checking for the total size of all fonts being <= 10MB will do. */ auto fonts = asset->font_data (); int total_size = 0; for (auto i: fonts) { total_size += i.second.size(); } if (total_size > 10 * 1024 * 1024) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TIMED_TEXT_FONTS_TOO_LARGE_IN_BYTES, *asset->file() ) ); } if (!asset->start_time()) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_START_TIME, *asset->file()) ); } else if (asset->start_time() != dcp::Time()) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::SUBTITLE_START_TIME_NON_ZERO, *asset->file()) ); } } static void verify_subtitle_asset ( shared_ptr asset, function)> stage, boost::filesystem::path xsd_dtd_directory, vector& notes, State& state ) { stage ("Checking subtitle XML", asset->file()); /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk * gets passed through libdcp which may clean up and therefore hide errors. */ validate_xml (asset->raw_xml(), xsd_dtd_directory, notes); auto smpte = dynamic_pointer_cast(asset); if (smpte) { verify_smpte_subtitle_asset (smpte, notes, state); } } static void verify_closed_caption_asset ( shared_ptr asset, function)> stage, boost::filesystem::path xsd_dtd_directory, vector& notes, State& state ) { verify_subtitle_asset (asset, stage, xsd_dtd_directory, notes, state); if (asset->raw_xml().size() > 256 * 1024) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_XML_TOO_LARGE_IN_BYTES, *asset->file() ) ); } } static void check_text_timing ( vector> reels, optional picture_frame_rate, vector& notes, std::function)> check, std::function)> xml, std::function)> duration ) { /* end of last subtitle (in editable units) */ optional last_out; auto too_short = false; auto too_close = false; auto too_early = false; /* current reel start time (in editable units) */ int64_t reel_offset = 0; std::function parse; parse = [&parse, &last_out, &too_short, &too_close, &too_early, &reel_offset](cxml::ConstNodePtr node, int tcr, int pfr, bool first_reel) { if (node->name() == "Subtitle") { dcp::Time in (node->string_attribute("TimeIn"), tcr); dcp::Time out (node->string_attribute("TimeOut"), tcr); if (first_reel && in < dcp::Time(0, 0, 4, 0, tcr)) { too_early = true; } auto length = out - in; if (length.as_editable_units(pfr) < 15) { too_short = true; } if (last_out) { /* XXX: this feels dubious - is it really what Bv2.1 means? */ auto distance = reel_offset + in.as_editable_units(pfr) - *last_out; if (distance >= 0 && distance < 2) { too_close = true; } } last_out = reel_offset + out.as_editable_units(pfr); } else { for (auto i: node->node_children()) { parse(i, tcr, pfr, first_reel); } } }; for (auto i = 0U; i < reels.size(); ++i) { if (!check(reels[i])) { continue; } /* We need to look at instances in the XML being checked, so we can't use the subtitles * read in by libdcp's parser. */ auto doc = make_shared("SubtitleReel"); doc->read_string (xml(reels[i])); auto const tcr = doc->number_child("TimeCodeRate"); parse (doc, tcr, picture_frame_rate.get_value_or(24), i == 0); reel_offset += duration(reels[i]); } if (too_early) { notes.push_back( VerificationNote( VerificationNote::VERIFY_WARNING, VerificationNote::FIRST_TEXT_TOO_EARLY ) ); } if (too_short) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_TOO_SHORT ) ); } if (too_close) { notes.push_back ( VerificationNote( VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_TOO_CLOSE ) ); } } struct LinesCharactersResult { bool warning_length_exceeded = false; bool error_length_exceeded = false; bool line_count_exceeded = false; }; static void check_text_lines_and_characters ( shared_ptr asset, int warning_length, int error_length, LinesCharactersResult* result ) { class Event { public: Event (dcp::Time time_, float position_, int characters_) : time (time_) , position (position_) , characters (characters_) {} Event (dcp::Time time_, shared_ptr start_) : time (time_) , start (start_) {} dcp::Time time; int position; //< position from 0 at top of screen to 100 at bottom int characters; shared_ptr start; }; vector> events; auto position = [](shared_ptr sub) { switch (sub->v_align()) { case VALIGN_TOP: return lrintf(sub->v_position() * 100); case VALIGN_CENTER: return lrintf((0.5f + sub->v_position()) * 100); case VALIGN_BOTTOM: return lrintf((1.0f - sub->v_position()) * 100); } return 0L; }; for (auto j: asset->subtitles()) { auto text = dynamic_pointer_cast(j); if (text) { auto in = make_shared(text->in(), position(text), text->text().length()); events.push_back(in); events.push_back(make_shared(text->out(), in)); } } std::sort(events.begin(), events.end(), [](shared_ptr const& a, shared_ptrconst& b) { return a->time < b->time; }); map current; for (auto i: events) { if (current.size() > 3) { result->line_count_exceeded = true; } for (auto j: current) { if (j.second >= warning_length) { result->warning_length_exceeded = true; } if (j.second >= error_length) { result->error_length_exceeded = true; } } if (i->start) { /* end of a subtitle */ DCP_ASSERT (current.find(i->start->position) != current.end()); if (current[i->start->position] == i->start->characters) { current.erase(i->start->position); } else { current[i->start->position] -= i->start->characters; } } else { /* start of a subtitle */ if (current.find(i->position) == current.end()) { current[i->position] = i->characters; } else { current[i->position] += i->characters; } } } } static void check_text_timing (vector> reels, vector& notes) { if (reels.empty()) { return; } optional picture_frame_rate; if (reels[0]->main_picture()) { picture_frame_rate = reels[0]->main_picture()->frame_rate().numerator; } if (reels[0]->main_subtitle()) { check_text_timing (reels, picture_frame_rate, notes, [](shared_ptr reel) { return static_cast(reel->main_subtitle()); }, [](shared_ptr reel) { return reel->main_subtitle()->asset()->raw_xml(); }, [](shared_ptr reel) { return reel->main_subtitle()->actual_duration(); } ); } for (auto i = 0U; i < reels[0]->closed_captions().size(); ++i) { check_text_timing (reels, picture_frame_rate, notes, [i](shared_ptr reel) { return i < reel->closed_captions().size(); }, [i](shared_ptr reel) { return reel->closed_captions()[i]->asset()->raw_xml(); }, [i](shared_ptr reel) { return reel->closed_captions()[i]->actual_duration(); } ); } } vector dcp::verify ( vector directories, function)> stage, function progress, boost::filesystem::path xsd_dtd_directory ) { xsd_dtd_directory = boost::filesystem::canonical (xsd_dtd_directory); vector notes; State state; vector> dcps; for (auto i: directories) { dcps.push_back (shared_ptr (new DCP (i))); } for (auto dcp: dcps) { stage ("Checking DCP", dcp->directory()); try { dcp->read (¬es); } catch (ReadError& e) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what()))); } catch (XMLError& e) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what()))); } catch (MXFFileError& e) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what()))); } catch (cxml::Error& e) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::GENERAL_READ, string(e.what()))); } if (dcp->standard() != dcp::SMPTE) { notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::NOT_SMPTE)); } for (auto cpl: dcp->cpls()) { stage ("Checking CPL", cpl->file()); validate_xml (cpl->file().get(), xsd_dtd_directory, notes); for (auto const& i: cpl->additional_subtitle_languages()) { verify_language_tag (i, notes); } if (cpl->release_territory()) { verify_language_tag (cpl->release_territory().get(), notes); } if (dcp->standard() == dcp::SMPTE) { if (!cpl->annotation_text()) { notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_ANNOTATION_TEXT_IN_CPL)); } else if (cpl->annotation_text().get() != cpl->content_title_text()) { notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::CPL_ANNOTATION_TEXT_DIFFERS_FROM_CONTENT_TITLE_TEXT)); } } /* Check that the CPL's hash corresponds to the PKL */ for (auto i: dcp->pkls()) { optional h = i->hash(cpl->id()); if (h && make_digest(ArrayData(*cpl->file())) != *h) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::CPL_HASH_INCORRECT)); } } /* set to true if any reel has a MainSubtitle */ auto have_main_subtitle = false; /* set to true if any reel has no MainSubtitle */ auto have_no_main_subtitle = false; /* fewest number of closed caption assets seen in a reel */ size_t fewest_closed_captions = SIZE_MAX; /* most number of closed caption assets seen in a reel */ size_t most_closed_captions = 0; for (auto reel: cpl->reels()) { stage ("Checking reel", optional()); for (auto i: reel->assets()) { if (i->duration() && (i->duration().get() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::DURATION_TOO_SMALL, i->id())); } if ((i->intrinsic_duration() * i->edit_rate().denominator / i->edit_rate().numerator) < 1) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INTRINSIC_DURATION_TOO_SMALL, i->id())); } auto mxf = dynamic_pointer_cast(i); if (mxf && !mxf->hash()) { notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_HASH, i->id()}); } } if (dcp->standard() == dcp::SMPTE) { boost::optional duration; for (auto i: reel->assets()) { if (!duration) { duration = i->actual_duration(); } else if (*duration != i->actual_duration()) { notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISMATCHED_ASSET_DURATION, i->id())); break; } } } if (reel->main_picture()) { /* Check reel stuff */ auto const frame_rate = reel->main_picture()->frame_rate(); if (frame_rate.denominator != 1 || (frame_rate.numerator != 24 && frame_rate.numerator != 25 && frame_rate.numerator != 30 && frame_rate.numerator != 48 && frame_rate.numerator != 50 && frame_rate.numerator != 60 && frame_rate.numerator != 96)) { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_RATE)); } /* Check asset */ if (reel->main_picture()->asset_ref().resolved()) { verify_main_picture_asset (dcp, reel->main_picture(), stage, progress, notes); } } if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) { verify_main_sound_asset (dcp, reel->main_sound(), stage, progress, notes); } if (reel->main_subtitle()) { verify_main_subtitle_reel (reel->main_subtitle(), notes); if (reel->main_subtitle()->asset_ref().resolved()) { verify_subtitle_asset (reel->main_subtitle()->asset(), stage, xsd_dtd_directory, notes, state); } have_main_subtitle = true; } else { have_no_main_subtitle = true; } for (auto i: reel->closed_captions()) { verify_closed_caption_reel (i, notes); if (i->asset_ref().resolved()) { verify_closed_caption_asset (i->asset(), stage, xsd_dtd_directory, notes, state); } } fewest_closed_captions = std::min (fewest_closed_captions, reel->closed_captions().size()); most_closed_captions = std::max (most_closed_captions, reel->closed_captions().size()); } if (dcp->standard() == dcp::SMPTE) { if (have_main_subtitle && have_no_main_subtitle) { notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MAIN_SUBTITLE_NOT_IN_ALL_REELS}); } if (fewest_closed_captions != most_closed_captions) { notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_ASSET_COUNTS_DIFFER}); } check_text_timing (cpl->reels(), notes); LinesCharactersResult result; for (auto reel: cpl->reels()) { if (reel->main_subtitle() && reel->main_subtitle()->asset()) { check_text_lines_and_characters (reel->main_subtitle()->asset(), 52, 79, &result); } } if (result.line_count_exceeded) { notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::TOO_MANY_SUBTITLE_LINES)); } if (result.error_length_exceeded) { notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_LINE_TOO_LONG)); } else if (result.warning_length_exceeded) { notes.push_back (VerificationNote(VerificationNote::VERIFY_WARNING, VerificationNote::SUBTITLE_LINE_LONGER_THAN_RECOMMENDED)); } result = LinesCharactersResult(); for (auto reel: cpl->reels()) { for (auto i: reel->closed_captions()) { if (i->asset()) { check_text_lines_and_characters (i->asset(), 32, 32, &result); } } } if (result.line_count_exceeded) { notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::TOO_MANY_CLOSED_CAPTION_LINES)); } if (result.error_length_exceeded) { notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::CLOSED_CAPTION_LINE_TOO_LONG)); } } } for (auto pkl: dcp->pkls()) { stage ("Checking PKL", pkl->file()); validate_xml (pkl->file().get(), xsd_dtd_directory, notes); } if (dcp->asset_map_path()) { stage ("Checking ASSETMAP", dcp->asset_map_path().get()); validate_xml (dcp->asset_map_path().get(), xsd_dtd_directory, notes); } else { notes.push_back (VerificationNote(VerificationNote::VERIFY_ERROR, VerificationNote::MISSING_ASSETMAP)); } } return notes; } string dcp::note_to_string (dcp::VerificationNote note) { switch (note.code()) { case dcp::VerificationNote::GENERAL_READ: return *note.note(); case dcp::VerificationNote::CPL_HASH_INCORRECT: return "The hash of the CPL in the PKL does not agree with the CPL file."; case dcp::VerificationNote::INVALID_PICTURE_FRAME_RATE: return "The picture in a reel has an invalid frame rate."; case dcp::VerificationNote::PICTURE_HASH_INCORRECT: return dcp::String::compose("The hash of the picture asset %1 does not agree with the PKL file.", note.file()->filename()); case dcp::VerificationNote::PKL_CPL_PICTURE_HASHES_DIFFER: return dcp::String::compose("The PKL and CPL hashes differ for the picture asset %1.", note.file()->filename()); case dcp::VerificationNote::SOUND_HASH_INCORRECT: return dcp::String::compose("The hash of the sound asset %1 does not agree with the PKL file.", note.file()->filename()); case dcp::VerificationNote::PKL_CPL_SOUND_HASHES_DIFFER: return dcp::String::compose("The PKL and CPL hashes differ for the sound asset %1.", note.file()->filename()); case dcp::VerificationNote::EMPTY_ASSET_PATH: return "The asset map contains an empty asset path."; case dcp::VerificationNote::MISSING_ASSET: return String::compose("The file for an asset in the asset map cannot be found; missing file is %1.", note.file()->filename()); case dcp::VerificationNote::MISMATCHED_STANDARD: return "The DCP contains both SMPTE and Interop parts."; case dcp::VerificationNote::XML_VALIDATION_ERROR: return String::compose("An XML file is badly formed: %1 (%2:%3)", note.note().get(), note.file()->filename(), note.line().get()); case dcp::VerificationNote::MISSING_ASSETMAP: return "No ASSETMAP or ASSETMAP.xml was found."; case dcp::VerificationNote::INTRINSIC_DURATION_TOO_SMALL: return String::compose("The intrinsic duration of an asset is less than 1 second long: %1", note.note().get()); case dcp::VerificationNote::DURATION_TOO_SMALL: return String::compose("The duration of an asset is less than 1 second long: %1", note.note().get()); case dcp::VerificationNote::PICTURE_FRAME_TOO_LARGE_IN_BYTES: return String::compose("The instantaneous bit rate of the picture asset %1 is larger than the limit of 250Mbit/s in at least one place.", note.file()->filename()); case dcp::VerificationNote::PICTURE_FRAME_NEARLY_TOO_LARGE_IN_BYTES: return String::compose("The instantaneous bit rate of the picture asset %1 is close to the limit of 250Mbit/s in at least one place.", note.file()->filename()); case dcp::VerificationNote::EXTERNAL_ASSET: return String::compose("An asset that this DCP refers to is not included in the DCP. It may be a VF. Missing asset is %1.", note.note().get()); case dcp::VerificationNote::NOT_SMPTE: return "This DCP does not use the SMPTE standard, which is required for Bv2.1 compliance."; case dcp::VerificationNote::BAD_LANGUAGE: return String::compose("The DCP specifies a language '%1' which does not conform to the RFC 5646 standard.", note.note().get()); case dcp::VerificationNote::PICTURE_ASSET_INVALID_SIZE_IN_PIXELS: return String::compose("A picture asset's size (%1) is not one of those allowed by Bv2.1 (2048x858, 1998x1080, 4096x1716 or 3996x2160)", note.note().get()); case dcp::VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_2K: return String::compose("A picture asset's frame rate (%1) is not one of those allowed for 2K DCPs by Bv2.1 (24, 25 or 48fps)", note.note().get()); case dcp::VerificationNote::PICTURE_ASSET_INVALID_FRAME_RATE_FOR_4K: return String::compose("A picture asset's frame rate (%1) is not 24fps as required for 4K DCPs by Bv2.1", note.note().get()); case dcp::VerificationNote::PICTURE_ASSET_4K_3D: return "3D 4K DCPs are not allowed by Bv2.1"; case dcp::VerificationNote::CLOSED_CAPTION_XML_TOO_LARGE_IN_BYTES: return String::compose("The XML for the closed caption asset %1 is longer than the 256KB maximum required by Bv2.1", note.file()->filename()); case dcp::VerificationNote::TIMED_TEXT_ASSET_TOO_LARGE_IN_BYTES: return String::compose("The total size of the timed text asset %1 is larger than the 115MB maximum required by Bv2.1", note.file()->filename()); case dcp::VerificationNote::TIMED_TEXT_FONTS_TOO_LARGE_IN_BYTES: return String::compose("The total size of the fonts in timed text asset %1 is larger than the 10MB maximum required by Bv2.1", note.file()->filename()); case dcp::VerificationNote::MISSING_SUBTITLE_LANGUAGE: return String::compose("The XML for a SMPTE subtitle asset has no tag, which is required by Bv2.1", note.file()->filename()); case dcp::VerificationNote::SUBTITLE_LANGUAGES_DIFFER: return String::compose("Some subtitle assets have different tags than others", note.file()->filename()); case dcp::VerificationNote::MISSING_SUBTITLE_START_TIME: return String::compose("The XML for a SMPTE subtitle asset has no tag, which is required by Bv2.1", note.file()->filename()); case dcp::VerificationNote::SUBTITLE_START_TIME_NON_ZERO: return String::compose("The XML for a SMPTE subtitle asset has a non-zero tag, which is disallowed by Bv2.1", note.file()->filename()); case dcp::VerificationNote::FIRST_TEXT_TOO_EARLY: return "The first subtitle or closed caption is less than 4 seconds from the start of the DCP."; case dcp::VerificationNote::SUBTITLE_TOO_SHORT: return "At least one subtitle is less than the minimum of 15 frames suggested by Bv2.1"; case dcp::VerificationNote::SUBTITLE_TOO_CLOSE: return "At least one pair of subtitles are separated by less than the the minimum of 2 frames suggested by Bv2.1"; case dcp::VerificationNote::TOO_MANY_SUBTITLE_LINES: return "There are more than 3 subtitle lines in at least one place in the DCP, which Bv2.1 advises against."; case dcp::VerificationNote::SUBTITLE_LINE_LONGER_THAN_RECOMMENDED: return "There are more than 52 characters in at least one subtitle line, which Bv2.1 advises against."; case dcp::VerificationNote::SUBTITLE_LINE_TOO_LONG: return "There are more than 79 characters in at least one subtitle line, which Bv2.1 strongly advises against."; case dcp::VerificationNote::TOO_MANY_CLOSED_CAPTION_LINES: return "There are more than 3 closed caption lines in at least one place, which is disallowed by Bv2.1"; case dcp::VerificationNote::CLOSED_CAPTION_LINE_TOO_LONG: return "There are more than 32 characters in at least one closed caption line, which is disallowed by Bv2.1"; case dcp::VerificationNote::INVALID_SOUND_FRAME_RATE: return "A sound asset has a sampling rate other than 48kHz, which is disallowed by Bv2.1"; case dcp::VerificationNote::MISSING_ANNOTATION_TEXT_IN_CPL: return "The CPL has no tag, which is required by Bv2.1"; case dcp::VerificationNote::CPL_ANNOTATION_TEXT_DIFFERS_FROM_CONTENT_TITLE_TEXT: return "The CPL's differs from its , which Bv2.1 advises against."; case dcp::VerificationNote::MISMATCHED_ASSET_DURATION: return "All assets in a reel do not have the same duration, which is required by Bv2.1"; case dcp::VerificationNote::MAIN_SUBTITLE_NOT_IN_ALL_REELS: return "At least one reel contains a subtitle asset, but some reel(s) do not"; case dcp::VerificationNote::CLOSED_CAPTION_ASSET_COUNTS_DIFFER: return "At least one reel has closed captions, but reels have different numbers of closed caption assets."; case dcp::VerificationNote::MISSING_SUBTITLE_ENTRY_POINT: return "Subtitle assets must have an tag."; case dcp::VerificationNote::SUBTITLE_ENTRY_POINT_NON_ZERO: return "Subtitle assets must have an of 0."; case dcp::VerificationNote::MISSING_CLOSED_CAPTION_ENTRY_POINT: return "Closed caption assets must have an tag."; case dcp::VerificationNote::CLOSED_CAPTION_ENTRY_POINT_NON_ZERO: return "Closed caption assets must have an of 0."; case dcp::VerificationNote::MISSING_HASH: return String::compose("An asset is missing a tag: %1", note.note().get()); } return ""; }