Add --no-asset-hash-check and --asset-hash-check-maximum-size (DoM #2444).v1.8.56

5 files changed, 79 insertions, 42 deletions

diff --git a/src/verify.cc b/src/verify.cc
index 2f8f0a53..822037ca 100644
--- a/src/verify.cc
+++ b/src/verify.cc
@@ -488,27 +488,32 @@ verify_main_picture_asset (
 	shared_ptr<const ReelPictureAsset> reel_asset,
 	function<void (string, optional<boost::filesystem::path>)> stage,
 	function<void (float)> progress,
+	VerificationOptions options,
 	vector<VerificationNote>& notes
 	)
 {
 	auto asset = reel_asset->asset();
 	auto const file = *asset->file();
-	stage ("Checking picture asset hash", file);
-	auto const r = verify_asset (dcp, reel_asset, progress);
-	switch (r) {
-		case VerifyAssetResult::BAD:
-			notes.push_back ({
-				VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_PICTURE_HASH, file
-			});
-			break;
-		case VerifyAssetResult::CPL_PKL_DIFFER:
-			notes.push_back ({
-				VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_PICTURE_HASHES, file
-			});
-			break;
-		default:
-			break;
+
+	if (options.check_asset_hashes && (!options.maximum_asset_size_for_hash_check || boost::filesystem::file_size(file) < *options.maximum_asset_size_for_hash_check)) {
+		stage ("Checking picture asset hash", file);
+		auto const r = verify_asset (dcp, reel_asset, progress);
+		switch (r) {
+			case VerifyAssetResult::BAD:
+				notes.push_back ({
+					VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_PICTURE_HASH, file
+				});
+				break;
+			case VerifyAssetResult::CPL_PKL_DIFFER:
+				notes.push_back ({
+					VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_PICTURE_HASHES, file
+				});
+				break;
+			default:
+				break;
+		}
 	}
+
 	stage ("Checking picture frame sizes", asset->file());
 	verify_picture_asset (reel_asset, file, notes, progress);
 
@@ -571,23 +576,26 @@ verify_main_sound_asset (
 	shared_ptr<const ReelSoundAsset> reel_asset,
 	function<void (string, optional<boost::filesystem::path>)> stage,
 	function<void (float)> progress,
+	VerificationOptions options,
 	vector<VerificationNote>& notes
 	)
 {
 	auto asset = reel_asset->asset();
 	auto const file = *asset->file();
 
-	stage("Checking sound asset hash", file);
-	auto const r = verify_asset (dcp, reel_asset, progress);
-	switch (r) {
-		case VerifyAssetResult::BAD:
-			notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_SOUND_HASH, file});
-			break;
-		case VerifyAssetResult::CPL_PKL_DIFFER:
-			notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_SOUND_HASHES, file});
-			break;
-		default:
-			break;
+	if (options.check_asset_hashes && (!options.maximum_asset_size_for_hash_check || boost::filesystem::file_size(file) < *options.maximum_asset_size_for_hash_check)) {
+		stage("Checking sound asset hash", file);
+		auto const r = verify_asset (dcp, reel_asset, progress);
+		switch (r) {
+			case VerifyAssetResult::BAD:
+				notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::INCORRECT_SOUND_HASH, file});
+				break;
+			case VerifyAssetResult::CPL_PKL_DIFFER:
+				notes.push_back({VerificationNote::Type::ERROR, VerificationNote::Code::MISMATCHED_SOUND_HASHES, file});
+				break;
+			default:
+				break;
+		}
 	}
 
 	stage ("Checking sound asset metadata", file);
@@ -1297,6 +1305,7 @@ verify_reel(
 	function<void (string, optional<boost::filesystem::path>)> stage,
 	boost::filesystem::path xsd_dtd_directory,
 	function<void (float)> progress,
+	VerificationOptions options,
 	vector<VerificationNote>& notes,
 	State& state,
 	bool* have_main_subtitle,
@@ -1350,7 +1359,7 @@ verify_reel(
 		}
 		/* Check asset */
 		if (reel->main_picture()->asset_ref().resolved()) {
-			verify_main_picture_asset(dcp, reel->main_picture(), stage, progress, notes);
+			verify_main_picture_asset(dcp, reel->main_picture(), stage, progress, options, notes);
 			auto const asset_size = reel->main_picture()->asset()->size();
 			if (main_picture_active_area) {
 				if (main_picture_active_area->width > asset_size.width) {
@@ -1374,7 +1383,7 @@ verify_reel(
 	}
 
 	if (reel->main_sound() && reel->main_sound()->asset_ref().resolved()) {
-		verify_main_sound_asset(dcp, reel->main_sound(), stage, progress, notes);
+		verify_main_sound_asset(dcp, reel->main_sound(), stage, progress, options, notes);
 	}
 
 	if (reel->main_subtitle()) {
@@ -1420,6 +1429,7 @@ verify_cpl(
 	function<void (string, optional<boost::filesystem::path>)> stage,
 	boost::filesystem::path xsd_dtd_directory,
 	function<void (float)> progress,
+	VerificationOptions options,
 	vector<VerificationNote>& notes,
 	State& state
 	)
@@ -1537,6 +1547,7 @@ verify_cpl(
 			stage,
 			xsd_dtd_directory,
 			progress,
+			options,
 			notes,
 			state,
 			&have_main_subtitle,
@@ -1695,6 +1706,7 @@ dcp::verify (
 	vector<boost::filesystem::path> directories,
 	function<void (string, optional<boost::filesystem::path>)> stage,
 	function<void (float)> progress,
+	VerificationOptions options,
 	optional<boost::filesystem::path> xsd_dtd_directory
 	)
 {
@@ -1744,6 +1756,7 @@ dcp::verify (
 				stage,
 				*xsd_dtd_directory,
 				progress,
+				options,
 				notes,
 				state
 				);
diff --git a/src/verify.h b/src/verify.h
index 1ddb7066..77fc28b3 100644
--- a/src/verify.h
+++ b/src/verify.h
@@ -488,10 +488,21 @@ private:
 };
 
 
+struct VerificationOptions
+{
+	///< If set, any assets larger than this number of bytes will not have their hashes checked
+	boost::optional<boost::uintmax_t> maximum_asset_size_for_hash_check;
+	///< true to check asset hashes (except those which match maximum_asset_size_for_hash_check)
+	///< false to check no asset hashes.
+	bool check_asset_hashes = true;
+};
+
+
 std::vector<VerificationNote> verify (
 	std::vector<boost::filesystem::path> directories,
 	boost::function<void (std::string, boost::optional<boost::filesystem::path>)> stage,
 	boost::function<void (float)> progress,
+	VerificationOptions options = {},
 	boost::optional<boost::filesystem::path> xsd_dtd_directory = boost::optional<boost::filesystem::path>()
 	);
 
diff --git a/test/combine_test.cc b/test/combine_test.cc
index 9fea304a..dd23b9ae 100644
--- a/test/combine_test.cc
+++ b/test/combine_test.cc
@@ -86,7 +86,7 @@ check_no_errors (boost::filesystem::path path)
 {
 	vector<boost::filesystem::path> directories;
 	directories.push_back (path);
-	auto notes = dcp::verify (directories, &stage, &progress, xsd_test);
+	auto notes = dcp::verify(directories, &stage, &progress, {}, xsd_test);
 	vector<dcp::VerificationNote> filtered_notes;
 	std::copy_if (notes.begin(), notes.end(), std::back_inserter(filtered_notes), [](dcp::VerificationNote const& i) {
 		return i.code() != dcp::VerificationNote::Code::INVALID_STANDARD && i.code() != dcp::VerificationNote::Code::INVALID_SUBTITLE_DURATION;
@@ -474,7 +474,7 @@ BOOST_AUTO_TEST_CASE(combine_multi_reel_subtitles)
 
 	check_combined({in}, out);
 
-	auto notes = dcp::verify({out}, &stage, &progress, xsd_test);
+	auto notes = dcp::verify({out}, &stage, &progress, {}, xsd_test);
 	vector<dcp::VerificationNote> filtered_notes;
 	std::copy_if(notes.begin(), notes.end(), std::back_inserter(filtered_notes), [](dcp::VerificationNote const& i) {
 		return i.code() != dcp::VerificationNote::Code::INVALID_STANDARD && i.code() != dcp::VerificationNote::Code::MISMATCHED_PKL_ANNOTATION_TEXT_WITH_CPL;
diff --git a/test/verify_test.cc b/test/verify_test.cc
index eaec1617..cde49830 100644
--- a/test/verify_test.cc
+++ b/test/verify_test.cc
@@ -297,7 +297,7 @@ static
 void
 check_verify_result (vector<path> dir, vector<dcp::VerificationNote> test_notes)
 {
-	auto notes = dcp::verify ({dir}, &stage, &progress, xsd_test);
+	auto notes = dcp::verify({dir}, &stage, &progress, {}, xsd_test);
 	std::sort (notes.begin(), notes.end());
 	std::sort (test_notes.begin(), test_notes.end());
 
@@ -345,7 +345,7 @@ check_verify_result_after_replace (string suffix, boost::function<path (string)>
 		e.replace (from, to);
 	}
 
-	auto notes = dcp::verify ({dir}, &stage, &progress, xsd_test);
+	auto notes = dcp::verify({dir}, &stage, &progress, {}, xsd_test);
 
 	BOOST_REQUIRE_EQUAL (notes.size(), codes.size());
 	auto i = notes.begin();
@@ -362,7 +362,7 @@ BOOST_AUTO_TEST_CASE (verify_no_error)
 {
 	stages.clear ();
 	auto dir = setup (1, "no_error");
-	auto notes = dcp::verify ({dir}, &stage, &progress, xsd_test);
+	auto notes = dcp::verify({dir}, &stage, &progress, {}, xsd_test);
 
 	path const cpl_file = dir / dcp_test1_cpl;
 	path const pkl_file = dir / dcp_test1_pkl;
@@ -603,7 +603,7 @@ BOOST_AUTO_TEST_CASE (verify_invalid_standard)
 {
 	stages.clear ();
 	auto dir = setup (3, "verify_invalid_standard");
-	auto notes = dcp::verify ({dir}, &stage, &progress, xsd_test);
+	auto notes = dcp::verify({dir}, &stage, &progress, {}, xsd_test);
 
 	path const cpl_file = dir / "cpl_cbfd2bc0-21cf-4a8f-95d8-9cddcbe51296.xml";
 	path const pkl_file = dir / "pkl_d87a950c-bd6f-41f6-90cc-56ccd673e131.xml";
@@ -1245,7 +1245,7 @@ check_picture_size (int width, int height, int frame_rate, bool three_d)
 	d->set_annotation_text("A Test DCP");
 	d->write_xml();
 
-	return dcp::verify ({dcp_path}, &stage, &progress, xsd_test);
+	return dcp::verify({dcp_path}, &stage, &progress, {}, xsd_test);
 }
 
 
diff --git a/tools/dcpverify.cc b/tools/dcpverify.cc
index 5847903f..23023fa3 100644
--- a/tools/dcpverify.cc
+++ b/tools/dcpverify.cc
@@ -34,6 +34,7 @@
 #include "verify.h"
 #include "compose.hpp"
 #include "common.h"
+#include "raw_convert.h"
 #include <boost/bind.hpp>
 #include <boost/optional.hpp>
 #include <boost/filesystem.hpp>
@@ -53,11 +54,13 @@ static void
 help (string n)
 {
 	cerr << "Syntax: " << n << " [OPTION] <DCP>\n"
-	     << "  -V, --version           show libdcp version\n"
-	     << "  -h, --help              show this help\n"
-	     << "  --ignore-missing-assets don't give errors about missing assets\n"
-	     << "  --ignore-bv21-smpte     don't give the SMPTE Bv2.1 error about a DCP not being SMPTE\n"
-	     << "  -q, --quiet             don't report progress\n";
+	     << "  -V, --version                               	show libdcp version\n"
+	     << "  -h, --help                                  	show this help\n"
+	     << "  --ignore-missing-assets                     	don't give errors about missing assets\n"
+	     << "  --ignore-bv21-smpte                         	don't give the SMPTE Bv2.1 error about a DCP not being SMPTE\n"
+	     << "  --no-asset-hash-check                       	don't check asset hashes\n"
+	     << "  --asset-hash-check-maximum-size <size-in-MB> only check hashes for assets smaller than this size (in MB)\n"
+	     << "  -q, --quiet                                  don't report progress\n";
 }
 
 
@@ -70,6 +73,8 @@ main (int argc, char* argv[])
 	bool ignore_bv21_smpte = false;
 	bool quiet = false;
 
+	dcp::VerificationOptions verification_options;
+
 	int option_index = 0;
 	while (true) {
 		static struct option long_options[] = {
@@ -77,11 +82,13 @@ main (int argc, char* argv[])
 			{ "help", no_argument, 0, 'h' },
 			{ "ignore-missing-assets", no_argument, 0, 'A' },
 			{ "ignore-bv21-smpte", no_argument, 0, 'B' },
+			{ "no-asset-hash-check", no_argument, 0, 'C' },
+			{ "asset-hash-check-maximum-size", required_argument, 0, 'D' },
 			{ "quiet", no_argument, 0, 'q' },
 			{ 0, 0, 0, 0 }
 		};
 
-		int c = getopt_long (argc, argv, "VhABq", long_options, &option_index);
+		int c = getopt_long (argc, argv, "VhABCD:q", long_options, &option_index);
 
 		if (c == -1) {
 			break;
@@ -102,6 +109,12 @@ main (int argc, char* argv[])
 		case 'B':
 			ignore_bv21_smpte = true;
 			break;
+		case 'C':
+			verification_options.check_asset_hashes = false;
+			break;
+		case 'D':
+			verification_options.maximum_asset_size_for_hash_check = dcp::raw_convert<int>(optarg) * 1000000LL;
+			break;
 		case 'q':
 			quiet = true;
 			break;
@@ -152,7 +165,7 @@ main (int argc, char* argv[])
 
 	vector<boost::filesystem::path> directories;
 	directories.push_back (argv[optind]);
-	auto notes = dcp::verify(directories, stage, progress);
+	auto notes = dcp::verify(directories, stage, progress, verification_options);
 	dcp::filter_notes (notes, ignore_missing_assets);
 
 	if (!quiet) {