X-Git-Url: https://git.carlh.net/gitweb/?a=blobdiff_plain;f=src%2Fverify.cc;h=167a7d8b49d78270b5f168e2d92267863bf1e6b4;hb=482a757103731501b22836b8c669d572ae3ab769;hp=0fcb88353979764a5d5a2d4749041e839f891237;hpb=961e8d6b0215f61ce0e39bedcf7d5b216336eae6;p=libdcp.git diff --git a/src/verify.cc b/src/verify.cc index 0fcb8835..167a7d8b 100644 --- a/src/verify.cc +++ b/src/verify.cc @@ -31,64 +31,79 @@ files in the program, then also delete it here. */ -#include "verify.h" -#include "dcp.h" + +/** @file src/verify.cc + * @brief dcp::verify() method and associated code + */ + + +#include "compose.hpp" #include "cpl.h" +#include "dcp.h" +#include "exceptions.h" +#include "filesystem.h" +#include "interop_subtitle_asset.h" +#include "mono_picture_asset.h" +#include "mono_picture_frame.h" +#include "raw_convert.h" #include "reel.h" #include "reel_closed_caption_asset.h" +#include "reel_interop_subtitle_asset.h" +#include "reel_markers_asset.h" #include "reel_picture_asset.h" #include "reel_sound_asset.h" +#include "reel_smpte_subtitle_asset.h" #include "reel_subtitle_asset.h" -#include "interop_subtitle_asset.h" -#include "mono_picture_asset.h" -#include "mono_picture_frame.h" +#include "smpte_subtitle_asset.h" #include "stereo_picture_asset.h" #include "stereo_picture_frame.h" -#include "exceptions.h" -#include "compose.hpp" -#include "raw_convert.h" -#include "reel_markers_asset.h" -#include "smpte_subtitle_asset.h" -#include -#include -#include -#include +#include "verify.h" +#include "verify_j2k.h" +#include +#include +#include +#include +#include +#include #include #include #include #include -#include -#include -#include -#include #include #include -#include -#include +#include #include #include -#include +#include +#include +#include +#include #include +#include #include +#include +#include #include -#include -using std::list; -using std::vector; -using std::string; + using std::cout; +using std::dynamic_pointer_cast; +using std::function; +using std::list; +using std::make_shared; using std::map; using std::max; using std::set; using std::shared_ptr; -using std::make_shared; +using std::string; +using std::vector; using boost::optional; -using boost::function; -using std::dynamic_pointer_cast; + using namespace dcp; using namespace xercesc; + static string xml_ch_to_string (XMLCh const * a) @@ -99,6 +114,7 @@ xml_ch_to_string (XMLCh const * a) return o; } + class XMLValidationError { public: @@ -144,22 +160,22 @@ private: class DCPErrorHandler : public ErrorHandler { public: - void warning(const SAXParseException& e) + void warning(const SAXParseException& e) override { maybe_add (XMLValidationError(e)); } - void error(const SAXParseException& e) + void error(const SAXParseException& e) override { maybe_add (XMLValidationError(e)); } - void fatalError(const SAXParseException& e) + void fatalError(const SAXParseException& e) override { maybe_add (XMLValidationError(e)); } - void resetErrors() { + void resetErrors() override { _errors.clear (); } @@ -184,7 +200,8 @@ private: list _errors; }; -class StringToXMLCh : public boost::noncopyable + +class StringToXMLCh { public: StringToXMLCh (string a) @@ -192,6 +209,9 @@ public: _buffer = XMLString::transcode(a.c_str()); } + StringToXMLCh (StringToXMLCh const&) = delete; + StringToXMLCh& operator= (StringToXMLCh const&) = delete; + ~StringToXMLCh () { XMLString::release (&_buffer); @@ -205,6 +225,7 @@ private: XMLCh* _buffer; }; + class LocalFileResolver : public EntityResolver { public: @@ -223,13 +244,14 @@ public: add("http://www.digicine.com/PROTO-ASDCP-AM-20040311.xsd", "PROTO-ASDCP-AM-20040311.xsd"); add("http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", "PROTO-ASDCP-CC-CPL-20070926.xsd"); add("interop-subs", "DCSubtitle.v1.mattsson.xsd"); - add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "SMPTE-428-7-2010-DCST.xsd"); + add("http://www.smpte-ra.org/schemas/428-7/2010/DCST.xsd", "DCDMSubtitle-2010.xsd"); + add("http://www.smpte-ra.org/schemas/428-7/2014/DCST.xsd", "DCDMSubtitle-2014.xsd"); add("http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", "SMPTE-429-16.xsd"); add("http://www.dolby.com/schemas/2012/AD", "Dolby-2012-AD.xsd"); add("http://www.smpte-ra.org/schemas/429-10/2008/Main-Stereo-Picture-CPL", "SMPTE-429-10-2008.xsd"); } - InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id) + InputSource* resolveEntity(XMLCh const *, XMLCh const * system_id) override { if (!system_id) { return 0; @@ -259,7 +281,7 @@ private: static void parse (XercesDOMParser& parser, boost::filesystem::path xml) { - parser.parse(xml.string().c_str()); + parser.parse(xml.c_str()); } @@ -271,9 +293,83 @@ parse (XercesDOMParser& parser, string xml) } +class Context +{ +public: + Context( + std::vector& notes_, + boost::filesystem::path xsd_dtd_directory_, + function)> stage_, + function progress_, + VerificationOptions options_ + ) + : notes(notes_) + , xsd_dtd_directory(xsd_dtd_directory_) + , stage(stage_) + , progress(progress_) + , options(options_) + { + + } + + Context(Context const&) = delete; + Context& operator=(Context const&) = delete; + + template + void ok(dcp::VerificationNote::Code code, Args... args) + { + add_note({dcp::VerificationNote::Type::OK, code, std::forward(args)...}); + } + + template + void warning(dcp::VerificationNote::Code code, Args... args) + { + add_note({dcp::VerificationNote::Type::WARNING, code, std::forward(args)...}); + } + + template + void bv21_error(dcp::VerificationNote::Code code, Args... args) + { + add_note({dcp::VerificationNote::Type::BV21_ERROR, code, std::forward(args)...}); + } + + template + void error(dcp::VerificationNote::Code code, Args... args) + { + add_note({dcp::VerificationNote::Type::ERROR, code, std::forward(args)...}); + } + + void add_note(dcp::VerificationNote note) + { + if (cpl) { + note.set_cpl_id(cpl->id()); + } + notes.push_back(std::move(note)); + } + + void add_note_if_not_existing(dcp::VerificationNote note) + { + if (find(notes.begin(), notes.end(), note) == notes.end()) { + add_note(note); + } + } + + std::vector& notes; + std::shared_ptr dcp; + std::shared_ptr cpl; + boost::filesystem::path xsd_dtd_directory; + function)> stage; + function progress; + VerificationOptions options; + + boost::optional subtitle_language; + boost::optional audio_channels; +}; + + template void -validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, vector& notes) +validate_xml(Context& context, T xml) { try { XMLPlatformUtils::Initialize (); @@ -302,6 +398,7 @@ validate_xml (T xml, boost::filesystem::path xsd_dtd_directory, vector dcp, shared_ptr reel_mxf, function progress) +verify_asset( + Context& context, + shared_ptr reel_file_asset, + string* reference_hash, + string* calculated_hash + ) { - auto const actual_hash = reel_mxf->asset_ref()->hash(progress); + DCP_ASSERT(reference_hash); + DCP_ASSERT(calculated_hash); + + /* When reading the DCP the hash will have been set to the one from the PKL/CPL. + * We want to calculate the hash of the actual file contents here, so that we + * can check it. unset_hash() means that this calculation will happen on the + * call to hash(). + */ + reel_file_asset->asset_ref()->unset_hash(); + *calculated_hash = reel_file_asset->asset_ref()->hash([&context](int64_t done, int64_t total) { + context.progress(float(done) / total); + }); - auto pkls = dcp->pkls(); + auto pkls = context.dcp->pkls(); /* We've read this DCP in so it must have at least one PKL */ DCP_ASSERT (!pkls.empty()); - auto asset = reel_mxf->asset_ref().asset(); + auto asset = reel_file_asset->asset_ref().asset(); - optional pkl_hash; + optional maybe_pkl_hash; for (auto i: pkls) { - pkl_hash = i->hash (reel_mxf->asset_ref()->id()); - if (pkl_hash) { + maybe_pkl_hash = i->hash (reel_file_asset->asset_ref()->id()); + if (maybe_pkl_hash) { break; } } - DCP_ASSERT (pkl_hash); + DCP_ASSERT(maybe_pkl_hash); + *reference_hash = *maybe_pkl_hash; - auto cpl_hash = reel_mxf->hash(); - if (cpl_hash && *cpl_hash != *pkl_hash) { - return VERIFY_ASSET_RESULT_CPL_PKL_DIFFER; + auto cpl_hash = reel_file_asset->hash(); + if (cpl_hash && *cpl_hash != *reference_hash) { + return VerifyAssetResult::CPL_PKL_DIFFER; } - if (actual_hash != *pkl_hash) { - return VERIFY_ASSET_RESULT_BAD; + if (*calculated_hash != *reference_hash) { + return VerifyAssetResult::BAD; } - return VERIFY_ASSET_RESULT_GOOD; + return VerifyAssetResult::GOOD; } -void -verify_language_tag (string tag, vector& notes) +static void +verify_language_tag(Context& context, string tag) { try { LanguageTag test (tag); } catch (LanguageTagError &) { - notes.push_back (VerificationNote(VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_LANGUAGE, tag)); + context.bv21_error(VerificationNote::Code::INVALID_LANGUAGE, tag); } } -enum VerifyPictureAssetResult -{ - VERIFY_PICTURE_ASSET_RESULT_GOOD, - VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE, - VERIFY_PICTURE_ASSET_RESULT_BAD, -}; - - -int -biggest_frame_size (shared_ptr frame) -{ - return frame->size (); -} - -int -biggest_frame_size (shared_ptr frame) -{ - return max(frame->left()->size(), frame->right()->size()); -} - - -template -optional -verify_picture_asset_type (shared_ptr reel_mxf, function progress) +static void +verify_picture_asset( + Context& context, + shared_ptr reel_file_asset, + boost::filesystem::path file, + int64_t start_frame + ) { - auto asset = dynamic_pointer_cast(reel_mxf->asset_ref().asset()); - if (!asset) { - return optional(); - } - - int biggest_frame = 0; - auto reader = asset->start_read (); + auto asset = dynamic_pointer_cast(reel_file_asset->asset_ref().asset()); auto const duration = asset->intrinsic_duration (); - for (int64_t i = 0; i < duration; ++i) { - shared_ptr frame = reader->get_frame (i); - biggest_frame = max(biggest_frame, biggest_frame_size(frame)); - progress (float(i) / duration); - } - static const int max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float())); - static const int risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float())); - if (biggest_frame > max_frame) { - return VERIFY_PICTURE_ASSET_RESULT_BAD; - } else if (biggest_frame > risky_frame) { - return VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE; - } + auto check_and_add = [&context](vector const& j2k_notes) { + for (auto i: j2k_notes) { + context.add_note_if_not_existing(i); + } + }; - return VERIFY_PICTURE_ASSET_RESULT_GOOD; -} + int const max_frame = rint(250 * 1000000 / (8 * asset->edit_rate().as_float())); + int const risky_frame = rint(230 * 1000000 / (8 * asset->edit_rate().as_float())); + + auto check_frame_size = [max_frame, risky_frame, file, start_frame](Context& context, int index, int size, int frame_rate) { + if (size > max_frame) { + context.add_note( + VerificationNote( + VerificationNote::Type::ERROR, VerificationNote::Code::INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file + ).set_frame(start_frame + index).set_frame_rate(frame_rate) + ); + } else if (size > risky_frame) { + context.add_note( + VerificationNote( + VerificationNote::Type::WARNING, VerificationNote::Code::NEARLY_INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file + ).set_frame(start_frame + index).set_frame_rate(frame_rate) + ); + } + }; + if (auto mono_asset = dynamic_pointer_cast(reel_file_asset->asset_ref().asset())) { + auto reader = mono_asset->start_read (); + for (int64_t i = 0; i < duration; ++i) { + auto frame = reader->get_frame (i); + check_frame_size(context, i, frame->size(), mono_asset->frame_rate().numerator); + if (!mono_asset->encrypted() || mono_asset->key()) { + vector j2k_notes; + verify_j2k(frame, start_frame, i, mono_asset->frame_rate().numerator, j2k_notes); + check_and_add (j2k_notes); + } + context.progress(float(i) / duration); + } + } else if (auto stereo_asset = dynamic_pointer_cast(asset)) { + auto reader = stereo_asset->start_read (); + for (int64_t i = 0; i < duration; ++i) { + auto frame = reader->get_frame (i); + check_frame_size(context, i, frame->left()->size(), stereo_asset->frame_rate().numerator); + check_frame_size(context, i, frame->right()->size(), stereo_asset->frame_rate().numerator); + if (!stereo_asset->encrypted() || stereo_asset->key()) { + vector j2k_notes; + verify_j2k(frame->left(), start_frame, i, stereo_asset->frame_rate().numerator, j2k_notes); + verify_j2k(frame->right(), start_frame, i, stereo_asset->frame_rate().numerator, j2k_notes); + check_and_add (j2k_notes); + } + context.progress(float(i) / duration); + } -static VerifyPictureAssetResult -verify_picture_asset (shared_ptr reel_mxf, function progress) -{ - auto r = verify_picture_asset_type(reel_mxf, progress); - if (!r) { - r = verify_picture_asset_type(reel_mxf, progress); } - - DCP_ASSERT (r); - return *r; } static void -verify_main_picture_asset ( - shared_ptr dcp, - shared_ptr reel_asset, - function)> stage, - function progress, - vector& notes - ) +verify_main_picture_asset(Context& context, shared_ptr reel_asset, int64_t start_frame) { auto asset = reel_asset->asset(); auto const file = *asset->file(); - stage ("Checking picture asset hash", file); - auto const r = verify_asset (dcp, reel_asset, progress); - switch (r) { - case VERIFY_ASSET_RESULT_BAD: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::INCORRECT_PICTURE_HASH, file - ) - ); - break; - case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::MISMATCHED_PICTURE_HASHES, file - ) - ); - break; - default: - break; - } - stage ("Checking picture frame sizes", asset->file()); - auto const pr = verify_picture_asset (reel_asset, progress); - switch (pr) { - case VERIFY_PICTURE_ASSET_RESULT_BAD: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file - ) - ); - break; - case VERIFY_PICTURE_ASSET_RESULT_FRAME_NEARLY_TOO_LARGE: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_WARNING, VerificationNote::NEARLY_INVALID_PICTURE_FRAME_SIZE_IN_BYTES, file - ) - ); - break; - default: - break; + + if (context.options.check_asset_hashes && (!context.options.maximum_asset_size_for_hash_check || filesystem::file_size(file) < *context.options.maximum_asset_size_for_hash_check)) { + context.stage("Checking picture asset hash", file); + string reference_hash; + string calculated_hash; + auto const r = verify_asset(context, reel_asset, &reference_hash, &calculated_hash); + switch (r) { + case VerifyAssetResult::BAD: + context.add_note( + dcp::VerificationNote( + VerificationNote::Type::ERROR, + VerificationNote::Code::INCORRECT_PICTURE_HASH, + file + ).set_reference_hash(reference_hash).set_calculated_hash(calculated_hash) + ); + break; + case VerifyAssetResult::CPL_PKL_DIFFER: + context.error(VerificationNote::Code::MISMATCHED_PICTURE_HASHES, file); + break; + default: + break; + } } + context.stage("Checking picture frame sizes", asset->file()); + verify_picture_asset(context, reel_asset, file, start_frame); + /* Only flat/scope allowed by Bv2.1 */ if ( asset->size() != Size(2048, 858) && asset->size() != Size(1998, 1080) && asset->size() != Size(4096, 1716) && asset->size() != Size(3996, 2160)) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::INVALID_PICTURE_SIZE_IN_PIXELS, - String::compose("%1x%2", asset->size().width, asset->size().height), - file - ) - ); + context.bv21_error(VerificationNote::Code::INVALID_PICTURE_SIZE_IN_PIXELS, String::compose("%1x%2", asset->size().width, asset->size().height), file); } /* Only 24, 25, 48fps allowed for 2K */ @@ -545,149 +632,133 @@ verify_main_picture_asset ( (asset->size() == Size(2048, 858) || asset->size() == Size(1998, 1080)) && (asset->edit_rate() != Fraction(24, 1) && asset->edit_rate() != Fraction(25, 1) && asset->edit_rate() != Fraction(48, 1)) ) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::INVALID_PICTURE_FRAME_RATE_FOR_2K, - String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), - file - ) - ); + context.bv21_error( + VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_2K, + String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), + file + ); } if (asset->size() == Size(4096, 1716) || asset->size() == Size(3996, 2160)) { /* Only 24fps allowed for 4K */ if (asset->edit_rate() != Fraction(24, 1)) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::INVALID_PICTURE_FRAME_RATE_FOR_4K, - String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), - file - ) - ); + context.bv21_error( + VerificationNote::Code::INVALID_PICTURE_FRAME_RATE_FOR_4K, + String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), + file + ); } /* Only 2D allowed for 4K */ if (dynamic_pointer_cast(asset)) { - notes.push_back( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, - VerificationNote::INVALID_PICTURE_ASSET_RESOLUTION_FOR_3D, - file - ) - ); + context.bv21_error( + VerificationNote::Code::INVALID_PICTURE_ASSET_RESOLUTION_FOR_3D, + String::compose("%1/%2", asset->edit_rate().numerator, asset->edit_rate().denominator), + file + ); } } - } static void -verify_main_sound_asset ( - shared_ptr dcp, - shared_ptr reel_asset, - function)> stage, - function progress, - vector& notes - ) +verify_main_sound_asset(Context& context, shared_ptr reel_asset) { auto asset = reel_asset->asset(); - stage ("Checking sound asset hash", asset->file()); - auto const r = verify_asset (dcp, reel_asset, progress); - switch (r) { - case VERIFY_ASSET_RESULT_BAD: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::INCORRECT_SOUND_HASH, *asset->file() - ) - ); - break; - case VERIFY_ASSET_RESULT_CPL_PKL_DIFFER: - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_ERROR, VerificationNote::MISMATCHED_SOUND_HASHES, *asset->file() - ) - ); - break; - default: - break; + auto const file = *asset->file(); + + if (context.options.check_asset_hashes && (!context.options.maximum_asset_size_for_hash_check || filesystem::file_size(file) < *context.options.maximum_asset_size_for_hash_check)) { + context.stage("Checking sound asset hash", file); + string reference_hash; + string calculated_hash; + auto const r = verify_asset(context, reel_asset, &reference_hash, &calculated_hash); + switch (r) { + case VerifyAssetResult::BAD: + context.add_note( + dcp::VerificationNote( + VerificationNote::Type::ERROR, + VerificationNote::Code::INCORRECT_SOUND_HASH, + file + ).set_reference_hash(reference_hash).set_calculated_hash(calculated_hash) + ); + break; + case VerifyAssetResult::CPL_PKL_DIFFER: + context.error(VerificationNote::Code::MISMATCHED_SOUND_HASHES, file); + break; + default: + break; + } + } + + if (!context.audio_channels) { + context.audio_channels = asset->channels(); + } else if (*context.audio_channels != asset->channels()) { + context.error(VerificationNote::Code::MISMATCHED_SOUND_CHANNEL_COUNTS, file); } - stage ("Checking sound asset metadata", asset->file()); + context.stage("Checking sound asset metadata", file); - verify_language_tag (asset->language(), notes); + if (auto lang = asset->language()) { + verify_language_tag(context, *lang); + } if (asset->sampling_rate() != 48000) { - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_SOUND_FRAME_RATE, *asset->file() - ) - ); + context.bv21_error(VerificationNote::Code::INVALID_SOUND_FRAME_RATE, raw_convert(asset->sampling_rate()), file); } } static void -verify_main_subtitle_reel (shared_ptr reel_asset, vector& notes) +verify_main_subtitle_reel(Context& context, shared_ptr reel_asset) { /* XXX: is Language compulsory? */ if (reel_asset->language()) { - verify_language_tag (*reel_asset->language(), notes); + verify_language_tag(context, *reel_asset->language()); } if (!reel_asset->entry_point()) { - notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_ENTRY_POINT }); + context.bv21_error(VerificationNote::Code::MISSING_SUBTITLE_ENTRY_POINT, reel_asset->id()); } else if (reel_asset->entry_point().get()) { - notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INCORRECT_SUBTITLE_ENTRY_POINT }); + context.bv21_error(VerificationNote::Code::INCORRECT_SUBTITLE_ENTRY_POINT, reel_asset->id()); } } static void -verify_closed_caption_reel (shared_ptr reel_asset, vector& notes) +verify_closed_caption_reel(Context& context, shared_ptr reel_asset) { /* XXX: is Language compulsory? */ if (reel_asset->language()) { - verify_language_tag (*reel_asset->language(), notes); + verify_language_tag(context, *reel_asset->language()); } if (!reel_asset->entry_point()) { - notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_CLOSED_CAPTION_ENTRY_POINT }); + context.bv21_error(VerificationNote::Code::MISSING_CLOSED_CAPTION_ENTRY_POINT, reel_asset->id()); } else if (reel_asset->entry_point().get()) { - notes.push_back ({VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INCORRECT_CLOSED_CAPTION_ENTRY_POINT }); + context.bv21_error(VerificationNote::Code::INCORRECT_CLOSED_CAPTION_ENTRY_POINT, reel_asset->id()); } } -struct State -{ - boost::optional subtitle_language; -}; - - - +/** Verify stuff that is common to both subtitles and closed captions */ void -verify_smpte_subtitle_asset ( +verify_smpte_timed_text_asset ( + Context& context, shared_ptr asset, - vector& notes, - State& state + optional reel_asset_duration ) { if (asset->language()) { - auto const language = *asset->language(); - verify_language_tag (language, notes); - if (!state.subtitle_language) { - state.subtitle_language = language; - } else if (state.subtitle_language != language) { - notes.push_back ({ VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISMATCHED_SUBTITLE_LANGUAGES }); - } + verify_language_tag(context, *asset->language()); } else { - notes.push_back ({ VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_LANGUAGE, *asset->file() }); + context.bv21_error(VerificationNote::Code::MISSING_SUBTITLE_LANGUAGE, *asset->file()); } - if (boost::filesystem::file_size(asset->file().get()) > 115 * 1024 * 1024) { - notes.push_back ({ VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_TIMED_TEXT_SIZE_IN_BYTES, *asset->file() }); + + auto const size = filesystem::file_size(asset->file().get()); + if (size > 115 * 1024 * 1024) { + context.bv21_error(VerificationNote::Code::INVALID_TIMED_TEXT_SIZE_IN_BYTES, raw_convert(size), *asset->file()); } + /* XXX: I'm not sure what Bv2.1_7.2.1 means when it says "the font resource shall not be larger than 10MB" * but I'm hoping that checking for the total size of all fonts being <= 10MB will do. */ @@ -697,69 +768,168 @@ verify_smpte_subtitle_asset ( total_size += i.second.size(); } if (total_size > 10 * 1024 * 1024) { - notes.push_back ({ VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_TIMED_TEXT_FONT_SIZE_IN_BYTES, asset->file().get() }); + context.bv21_error(VerificationNote::Code::INVALID_TIMED_TEXT_FONT_SIZE_IN_BYTES, raw_convert(total_size), asset->file().get()); } if (!asset->start_time()) { - notes.push_back ({ VerificationNote::VERIFY_BV21_ERROR, VerificationNote::MISSING_SUBTITLE_START_TIME, asset->file().get() }); + context.bv21_error(VerificationNote::Code::MISSING_SUBTITLE_START_TIME, asset->file().get()); } else if (asset->start_time() != Time()) { - notes.push_back ({ VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_SUBTITLE_START_TIME, asset->file().get() }); + context.bv21_error(VerificationNote::Code::INVALID_SUBTITLE_START_TIME, asset->file().get()); + } + + if (reel_asset_duration && *reel_asset_duration != asset->intrinsic_duration()) { + context.bv21_error( + VerificationNote::Code::MISMATCHED_TIMED_TEXT_DURATION, + String::compose("%1 %2", *reel_asset_duration, asset->intrinsic_duration()), + asset->file().get() + ); + } +} + + +/** Verify Interop subtitle / CCAP stuff */ +void +verify_interop_text_asset(Context& context, shared_ptr asset) +{ + if (asset->subtitles().empty()) { + context.error(VerificationNote::Code::MISSING_SUBTITLE, asset->id(), asset->file().get()); + } + auto const unresolved = asset->unresolved_fonts(); + if (!unresolved.empty()) { + context.error(VerificationNote::Code::MISSING_FONT, unresolved.front()); + } +} + + +/** Verify SMPTE subtitle-only stuff */ +void +verify_smpte_subtitle_asset(Context& context, shared_ptr asset) +{ + if (asset->language()) { + if (!context.subtitle_language) { + context.subtitle_language = *asset->language(); + } else if (context.subtitle_language != *asset->language()) { + context.bv21_error(VerificationNote::Code::MISMATCHED_SUBTITLE_LANGUAGES); + } + } + + DCP_ASSERT (asset->resource_id()); + auto xml_id = asset->xml_id(); + if (xml_id) { + if (asset->resource_id().get() != xml_id) { + context.bv21_error(VerificationNote::Code::MISMATCHED_TIMED_TEXT_RESOURCE_ID); + } + + if (asset->id() == asset->resource_id().get() || asset->id() == xml_id) { + context.bv21_error(VerificationNote::Code::INCORRECT_TIMED_TEXT_ASSET_ID); + } + } else { + context.warning(VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED); + } + + if (asset->raw_xml()) { + /* Deluxe require this in their QC even if it seems never to be mentioned in any standard */ + cxml::Document doc("SubtitleReel"); + doc.read_string(*asset->raw_xml()); + auto issue_date = doc.string_child("IssueDate"); + std::regex reg("^\\d\\d\\d\\d-\\d\\d-\\d\\dT\\d\\d:\\d\\d:\\d\\d$"); + if (!std::regex_match(issue_date, reg)) { + context.warning(VerificationNote::Code::INVALID_SUBTITLE_ISSUE_DATE, issue_date); + } } } +/** Verify all subtitle stuff */ static void -verify_subtitle_asset ( - shared_ptr asset, - function)> stage, - boost::filesystem::path xsd_dtd_directory, - vector& notes, - State& state - ) +verify_subtitle_asset(Context& context, shared_ptr asset, optional reel_asset_duration) { - stage ("Checking subtitle XML", asset->file()); + context.stage("Checking subtitle XML", asset->file()); /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk * gets passed through libdcp which may clean up and therefore hide errors. */ - validate_xml (asset->raw_xml(), xsd_dtd_directory, notes); + if (asset->raw_xml()) { + validate_xml(context, asset->raw_xml().get()); + } else { + context.warning(VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED); + } + + auto namespace_count = [](shared_ptr asset, string root_node) { + cxml::Document doc(root_node); + doc.read_string(asset->raw_xml().get()); + auto root = dynamic_cast(doc.node())->cobj(); + int count = 0; + for (auto ns = root->nsDef; ns != nullptr; ns = ns->next) { + ++count; + } + return count; + }; + + auto interop = dynamic_pointer_cast(asset); + if (interop) { + verify_interop_text_asset(context, interop); + if (namespace_count(asset, "DCSubtitle") > 1) { + context.warning(VerificationNote::Code::INCORRECT_SUBTITLE_NAMESPACE_COUNT, asset->id()); + } + } auto smpte = dynamic_pointer_cast(asset); if (smpte) { - verify_smpte_subtitle_asset (smpte, notes, state); + verify_smpte_timed_text_asset(context, smpte, reel_asset_duration); + verify_smpte_subtitle_asset(context, smpte); + /* This asset may be encrypted and in that case we'll have no raw_xml() */ + if (asset->raw_xml() && namespace_count(asset, "SubtitleReel") > 1) { + context.warning(VerificationNote::Code::INCORRECT_SUBTITLE_NAMESPACE_COUNT, asset->id()); + } } } +/** Verify all closed caption stuff */ static void verify_closed_caption_asset ( + Context& context, shared_ptr asset, - function)> stage, - boost::filesystem::path xsd_dtd_directory, - vector& notes, - State& state + optional reel_asset_duration ) { - verify_subtitle_asset (asset, stage, xsd_dtd_directory, notes, state); + context.stage("Checking closed caption XML", asset->file()); + /* Note: we must not use SubtitleAsset::xml_as_string() here as that will mean the data on disk + * gets passed through libdcp which may clean up and therefore hide errors. + */ + auto raw_xml = asset->raw_xml(); + if (raw_xml) { + validate_xml(context, *raw_xml); + if (raw_xml->size() > 256 * 1024) { + context.bv21_error(VerificationNote::Code::INVALID_CLOSED_CAPTION_XML_SIZE_IN_BYTES, raw_convert(raw_xml->size()), *asset->file()); + } + } else { + context.warning(VerificationNote::Code::MISSED_CHECK_OF_ENCRYPTED); + } - if (asset->raw_xml().size() > 256 * 1024) { - notes.push_back ( - VerificationNote( - VerificationNote::VERIFY_BV21_ERROR, VerificationNote::INVALID_CLOSED_CAPTION_XML_SIZE_IN_BYTES, *asset->file() - ) - ); + auto interop = dynamic_pointer_cast(asset); + if (interop) { + verify_interop_text_asset(context, interop); + } + + auto smpte = dynamic_pointer_cast(asset); + if (smpte) { + verify_smpte_timed_text_asset(context, smpte, reel_asset_duration); } } +/** Check the timing of the individual subtitles and make sure there are no empty nodes etc. */ static void -check_text_timing ( +verify_text_details ( + Context& context, vector> reels, - optional picture_frame_rate, - vector& notes, + int edit_rate, std::function)> check, - std::function)> xml, - std::function)> duration + std::function (shared_ptr)> xml, + std::function)> duration, + std::function)> id ) { /* end of last subtitle (in editable units) */ @@ -767,34 +937,79 @@ check_text_timing ( auto too_short = false; auto too_close = false; auto too_early = false; + auto reel_overlap = false; + auto empty_text = false; /* current reel start time (in editable units) */ int64_t reel_offset = 0; - - std::function parse; - parse = [&parse, &last_out, &too_short, &too_close, &too_early, &reel_offset](cxml::ConstNodePtr node, int tcr, int pfr, bool first_reel) { + optional missing_load_font_id; + + std::function, optional