Hand-apply bbfb370d7de28ec1e8f307865cc6253bb5d4366e from master; quicker digest calculation.

21 files changed, 90 insertions, 84 deletions

diff --git a/src/lib/audio_content.cc b/src/lib/audio_content.cc
index 61357c60f..9909e3a08 100644
--- a/src/lib/audio_content.cc
+++ b/src/lib/audio_content.cc
@@ -181,7 +181,7 @@ AudioContent::audio_analysis_path () const
 	}
 
 	boost::filesystem::path p = film->audio_analysis_dir ();
-	p /= digest().get_value_or ("X") + "_" + audio_mapping().digest();
+	p /= digest() + "_" + audio_mapping().digest();
 	return p;
 }
 
diff --git a/src/lib/content.cc b/src/lib/content.cc
index a8d058cc1..550a8cd05 100644
--- a/src/lib/content.cc
+++ b/src/lib/content.cc
@@ -88,7 +88,7 @@ Content::Content (shared_ptr<const Film> f, cxml::ConstNodePtr node)
 	for (list<cxml::NodePtr>::const_iterator i = path_children.begin(); i != path_children.end(); ++i) {
 		_paths.push_back ((*i)->content ());
 	}
-	_digest = node->optional_string_child ("Digest");
+	_digest = node->optional_string_child ("Digest").get_value_or ("X");
 	_position = DCPTime (node->number_child<double> ("Position"));
 	_trim_start = DCPTime (node->number_child<double> ("TrimStart"));
 	_trim_end = DCPTime (node->number_child<double> ("TrimEnd"));
@@ -124,21 +124,15 @@ Content::as_xml (xmlpp::Node* node) const
 	for (vector<boost::filesystem::path>::const_iterator i = _paths.begin(); i != _paths.end(); ++i) {
 		node->add_child("Path")->add_child_text (i->string ());
 	}
-	if (_digest) {
-		node->add_child("Digest")->add_child_text (_digest.get ());
-	}
+	node->add_child("Digest")->add_child_text (_digest);
 	node->add_child("Position")->add_child_text (raw_convert<string> (_position.get ()));
 	node->add_child("TrimStart")->add_child_text (raw_convert<string> (_trim_start.get ()));
 	node->add_child("TrimEnd")->add_child_text (raw_convert<string> (_trim_end.get ()));
 }
 
 void
-Content::examine (shared_ptr<Job> job, bool calculate_digest)
+Content::examine (shared_ptr<Job> job)
 {
-	if (!calculate_digest) {
-		return;
-	}
-
 	if (job) {
 		job->sub (_("Computing digest"));
 	}
@@ -146,8 +140,12 @@ Content::examine (shared_ptr<Job> job, bool calculate_digest)
 	boost::mutex::scoped_lock lm (_mutex);
 	vector<boost::filesystem::path> p = _paths;
 	lm.unlock ();
-	
-	string const d = md5_digest (p, job);
+
+	/* Some content files are very big, so we use a poor's
+	   digest here: a MD5 of the first and last 1e6 bytes with the
+	   size of the first file tacked on the end as a string.
+	*/
+	string const d = md5_digest_head_tail (p, 1000000) + dcp::raw_convert<string> (boost::filesystem::file_size (p.front ()));
 
 	lm.lock ();
 	_digest = d;
@@ -220,7 +218,7 @@ Content::clone () const
 string
 Content::technical_summary () const
 {
-	return String::compose ("%1 %2 %3", path_summary(), digest().get_value_or("X"), position().seconds());
+	return String::compose ("%1 %2 %3", path_summary(), digest(), position().seconds());
 }
 
 DCPTime
@@ -237,7 +235,7 @@ Content::identifier () const
 {
 	SafeStringStream s;
 	
-	s << Content::digest().get_value_or("X")
+	s << Content::digest()
 	  << "_" << position().get()
 	  << "_" << trim_start().get()
 	  << "_" << trim_end().get();
diff --git a/src/lib/content.h b/src/lib/content.h
index a3e6da988..c6cede5fa 100644
--- a/src/lib/content.h
+++ b/src/lib/content.h
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -66,9 +66,8 @@ public:
 	/** Examine the content to establish digest, frame rates and any other
 	 *  useful metadata.
 	 *  @param job Job to use to report progress, or 0.
-	 *  @param calculate_digest True to calculate a digest for the content's file(s).
 	 */
-	virtual void examine (boost::shared_ptr<Job> job, bool calculate_digest);
+	virtual void examine (boost::shared_ptr<Job> job);
 	
 	/** @return Quick one-line summary of the content, as will be presented in the
 	 *  film editor.
@@ -107,8 +106,11 @@ public:
 	
 	bool paths_valid () const;
 
-	/** @return MD5 digest of the content's file(s) */
-	boost::optional<std::string> digest () const {
+       /** @return Digest of the content's file(s).  Note: this is
+        *  not a complete MD5-or-whatever hash, but a sort of poor
+        *  man' version (see comments in ::examine).
+        */
+	std::string digest () const {
 		boost::mutex::scoped_lock lm (_mutex);
 		return _digest;
 	}
@@ -167,7 +169,7 @@ protected:
 	std::vector<boost::filesystem::path> _paths;
 	
 private:
-	boost::optional<std::string> _digest;
+	std::string _digest;
 	DCPTime _position;
 	DCPTime _trim_start;
 	DCPTime _trim_end;
diff --git a/src/lib/dcp_content.cc b/src/lib/dcp_content.cc
index 2bf14dcff..d1a658001 100644
--- a/src/lib/dcp_content.cc
+++ b/src/lib/dcp_content.cc
@@ -77,12 +77,12 @@ DCPContent::read_directory (boost::filesystem::path p)
 }
 
 void
-DCPContent::examine (shared_ptr<Job> job, bool calculate_digest)
+DCPContent::examine (shared_ptr<Job> job)
 {
 	bool const could_be_played = can_be_played ();
 		
 	job->set_progress_unknown ();
-	Content::examine (job, calculate_digest);
+	Content::examine (job);
 	
 	shared_ptr<DCPExaminer> examiner (new DCPExaminer (shared_from_this ()));
 	take_from_video_examiner (examiner);
diff --git a/src/lib/dcp_content.h b/src/lib/dcp_content.h
index 82f5a8089..aa53d76a9 100644
--- a/src/lib/dcp_content.h
+++ b/src/lib/dcp_content.h
@@ -52,7 +52,7 @@ public:
 
 	DCPTime full_length () const;
 	
-	void examine (boost::shared_ptr<Job>, bool calculate_digest);
+	void examine (boost::shared_ptr<Job>);
 	std::string summary () const;
 	std::string technical_summary () const;
 	void as_xml (xmlpp::Node *) const;
diff --git a/src/lib/dcp_subtitle_content.cc b/src/lib/dcp_subtitle_content.cc
index 45c4be9b2..9f2ff61c5 100644
--- a/src/lib/dcp_subtitle_content.cc
+++ b/src/lib/dcp_subtitle_content.cc
@@ -47,9 +47,9 @@ DCPSubtitleContent::DCPSubtitleContent (shared_ptr<const Film> film, cxml::Const
 }
 
 void
-DCPSubtitleContent::examine (shared_ptr<Job> job, bool calculate_digest)
+DCPSubtitleContent::examine (shared_ptr<Job> job)
 {
-	Content::examine (job, calculate_digest);
+	Content::examine (job);
 
 	shared_ptr<dcp::SubtitleContent> sc = load (path (0));
 	
diff --git a/src/lib/dcp_subtitle_content.h b/src/lib/dcp_subtitle_content.h
index 4b5f1fa05..05af71690 100644
--- a/src/lib/dcp_subtitle_content.h
+++ b/src/lib/dcp_subtitle_content.h
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2014 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2014-2015 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -27,7 +27,7 @@ public:
 	DCPSubtitleContent (boost::shared_ptr<const Film>, cxml::ConstNodePtr, int);
 
 	/* Content */
-	void examine (boost::shared_ptr<Job>, bool calculate_digest);
+	void examine (boost::shared_ptr<Job>);
 	std::string summary () const;
 	std::string technical_summary () const;
 	void as_xml (xmlpp::Node *) const;
diff --git a/src/lib/examine_content_job.cc b/src/lib/examine_content_job.cc
index ee887271f..2b8f118f8 100644
--- a/src/lib/examine_content_job.cc
+++ b/src/lib/examine_content_job.cc
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2012 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2012-2015 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -29,10 +29,9 @@ using std::string;
 using std::cout;
 using boost::shared_ptr;
 
-ExamineContentJob::ExamineContentJob (shared_ptr<const Film> f, shared_ptr<Content> c, bool calculate_digest)
+ExamineContentJob::ExamineContentJob (shared_ptr<const Film> f, shared_ptr<Content> c)
 	: Job (f)
 	, _content (c)
-	, _calculate_digest (calculate_digest)
 {
 
 }
@@ -50,7 +49,7 @@ ExamineContentJob::name () const
 void
 ExamineContentJob::run ()
 {
-	_content->examine (shared_from_this (), _calculate_digest);
+	_content->examine (shared_from_this ());
 	set_progress (1);
 	set_state (FINISHED_OK);
 }
diff --git a/src/lib/examine_content_job.h b/src/lib/examine_content_job.h
index e59dba0b8..016a56371 100644
--- a/src/lib/examine_content_job.h
+++ b/src/lib/examine_content_job.h
@@ -26,7 +26,7 @@ class Log;
 class ExamineContentJob : public Job
 {
 public:
-	ExamineContentJob (boost::shared_ptr<const Film>, boost::shared_ptr<Content>, bool calculate_digest);
+	ExamineContentJob (boost::shared_ptr<const Film>, boost::shared_ptr<Content>);
 	~ExamineContentJob ();
 
 	std::string name () const;
@@ -34,6 +34,5 @@ public:
 
 private:
 	boost::shared_ptr<Content> _content;
-	bool _calculate_digest;
 };
 
diff --git a/src/lib/ffmpeg_content.cc b/src/lib/ffmpeg_content.cc
index d5fd592d6..9e9473935 100644
--- a/src/lib/ffmpeg_content.cc
+++ b/src/lib/ffmpeg_content.cc
@@ -163,11 +163,11 @@ FFmpegContent::as_xml (xmlpp::Node* node) const
 }
 
 void
-FFmpegContent::examine (shared_ptr<Job> job, bool calculate_digest)
+FFmpegContent::examine (shared_ptr<Job> job)
 {
 	job->set_progress_unknown ();
 
-	Content::examine (job, calculate_digest);
+	Content::examine (job);
 
 	shared_ptr<FFmpegExaminer> examiner (new FFmpegExaminer (shared_from_this (), job));
 	take_from_video_examiner (examiner);
@@ -367,7 +367,7 @@ FFmpegContent::audio_analysis_path () const
 	*/
 
 	boost::filesystem::path p = film->audio_analysis_dir ();
-	string name = digest().get_value_or ("X");
+	string name = digest();
 	if (audio_stream ()) {
 		name += "_" + audio_stream()->identifier ();
 	}
diff --git a/src/lib/ffmpeg_content.h b/src/lib/ffmpeg_content.h
index 2831c2acf..76ba43567 100644
--- a/src/lib/ffmpeg_content.h
+++ b/src/lib/ffmpeg_content.h
@@ -56,7 +56,7 @@ public:
 		return boost::dynamic_pointer_cast<FFmpegContent> (Content::shared_from_this ());
 	}
 	
-	void examine (boost::shared_ptr<Job>, bool calculate_digest);
+	void examine (boost::shared_ptr<Job>);
 	std::string summary () const;
 	std::string technical_summary () const;
 	void as_xml (xmlpp::Node *) const;
diff --git a/src/lib/film.cc b/src/lib/film.cc
index 82add67c5..ecc98d7dc 100644
--- a/src/lib/film.cc
+++ b/src/lib/film.cc
@@ -952,20 +952,20 @@ Film::content () const
 }
 
 void
-Film::examine_content (shared_ptr<Content> c, bool calculate_digest)
+Film::examine_content (shared_ptr<Content> c)
 {
-	shared_ptr<Job> j (new ExamineContentJob (shared_from_this(), c, calculate_digest));
+	shared_ptr<Job> j (new ExamineContentJob (shared_from_this(), c));
 	JobManager::instance()->add (j);
 }
 
 void
-Film::examine_and_add_content (shared_ptr<Content> c, bool calculate_digest)
+Film::examine_and_add_content (shared_ptr<Content> c)
 {
 	if (dynamic_pointer_cast<FFmpegContent> (c)) {
 		run_ffprobe (c->path(0), file ("ffprobe.log"), _log);
 	}
 			
-	shared_ptr<Job> j (new ExamineContentJob (shared_from_this(), c, calculate_digest));
+	shared_ptr<Job> j (new ExamineContentJob (shared_from_this(), c));
 	j->Finished.connect (bind (&Film::maybe_add_content, this, boost::weak_ptr<Job> (j), boost::weak_ptr<Content> (c)));
 	JobManager::instance()->add (j);
 }
diff --git a/src/lib/film.h b/src/lib/film.h
index 43f7bae78..2de065159 100644
--- a/src/lib/film.h
+++ b/src/lib/film.h
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2012-2014 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2012-2015 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -252,8 +252,8 @@ public:
 	void set_directory (boost::filesystem::path);
 	void set_name (std::string);
 	void set_use_isdcf_name (bool);
-	void examine_content (boost::shared_ptr<Content>, bool calculate_digest);
-	void examine_and_add_content (boost::shared_ptr<Content>, bool calculate_digest);
+	void examine_content (boost::shared_ptr<Content>);
+	void examine_and_add_content (boost::shared_ptr<Content>);
 	void add_content (boost::shared_ptr<Content>);
 	void remove_content (boost::shared_ptr<Content>);
 	void move_content_earlier (boost::shared_ptr<Content>);
diff --git a/src/lib/image_content.cc b/src/lib/image_content.cc
index b8d2a6921..1dbd7871b 100644
--- a/src/lib/image_content.cc
+++ b/src/lib/image_content.cc
@@ -100,9 +100,9 @@ ImageContent::as_xml (xmlpp::Node* node) const
 }
 
 void
-ImageContent::examine (shared_ptr<Job> job, bool calculate_digest)
+ImageContent::examine (shared_ptr<Job> job)
 {
-	Content::examine (job, calculate_digest);
+	Content::examine (job);
 
 	shared_ptr<const Film> film = _film.lock ();
 	DCPOMATIC_ASSERT (film);
diff --git a/src/lib/image_content.h b/src/lib/image_content.h
index 8c50fd8a8..38aa77bf5 100644
--- a/src/lib/image_content.h
+++ b/src/lib/image_content.h
@@ -37,7 +37,7 @@ public:
 		return boost::dynamic_pointer_cast<ImageContent> (Content::shared_from_this ());
 	};
 
-	void examine (boost::shared_ptr<Job>, bool calculate_digest);
+	void examine (boost::shared_ptr<Job>);
 	std::string summary () const;
 	std::string technical_summary () const;
 	void as_xml (xmlpp::Node *) const;
diff --git a/src/lib/sndfile_content.cc b/src/lib/sndfile_content.cc
index e0b7cc10c..e757c77c5 100644
--- a/src/lib/sndfile_content.cc
+++ b/src/lib/sndfile_content.cc
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2013-2014 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2013-2015 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -82,10 +82,10 @@ SndfileContent::valid_file (boost::filesystem::path f)
 }
 
 void
-SndfileContent::examine (shared_ptr<Job> job, bool calculate_digest)
+SndfileContent::examine (shared_ptr<Job> job)
 {
 	job->set_progress_unknown ();
-	Content::examine (job, calculate_digest);
+	Content::examine (job);
 	shared_ptr<AudioExaminer> dec (new SndfileDecoder (shared_from_this()));
 	take_from_audio_examiner (dec);
 }
diff --git a/src/lib/sndfile_content.h b/src/lib/sndfile_content.h
index 1fff01f60..1bac51167 100644
--- a/src/lib/sndfile_content.h
+++ b/src/lib/sndfile_content.h
@@ -41,7 +41,7 @@ public:
 	
 	DCPTime full_length () const;
 	
-	void examine (boost::shared_ptr<Job>, bool calculate_digest);
+	void examine (boost::shared_ptr<Job>);
 	std::string summary () const;
 	std::string technical_summary () const;
 	std::string information () const;
diff --git a/src/lib/subrip_content.cc b/src/lib/subrip_content.cc
index 819bca321..fc0637bfd 100644
--- a/src/lib/subrip_content.cc
+++ b/src/lib/subrip_content.cc
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2014 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2014-2015 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -50,9 +50,9 @@ SubRipContent::SubRipContent (shared_ptr<const Film> film, cxml::ConstNodePtr no
 }
 
 void
-SubRipContent::examine (boost::shared_ptr<Job> job, bool calculate_digest)
+SubRipContent::examine (boost::shared_ptr<Job> job)
 {
-	Content::examine (job, calculate_digest);
+	Content::examine (job);
 	SubRip s (shared_from_this ());
 
 	shared_ptr<const Film> film = _film.lock ();
diff --git a/src/lib/subrip_content.h b/src/lib/subrip_content.h
index 4321ecb6a..e63526179 100644
--- a/src/lib/subrip_content.h
+++ b/src/lib/subrip_content.h
@@ -1,5 +1,5 @@
 /*
-    Copyright (C) 2014 Carl Hetherington <cth@carlh.net>
+    Copyright (C) 2014-2015 Carl Hetherington <cth@carlh.net>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -30,7 +30,7 @@ public:
 	}
 	
 	/* Content */
-	void examine (boost::shared_ptr<Job>, bool calculate_digest);
+	void examine (boost::shared_ptr<Job>);
 	std::string summary () const;
 	std::string technical_summary () const;
 	void as_xml (xmlpp::Node *) const;
diff --git a/src/lib/util.cc b/src/lib/util.cc
index c0e32b778..0a6f381db 100644
--- a/src/lib/util.cc
+++ b/src/lib/util.cc
@@ -417,45 +417,53 @@ dcpomatic_setup_gettext_i18n (string lang)
 #endif
 }
 
-/** @param job Optional job for which to report progress */
+/** Compute a digest of the first and last `size' bytes of a set of files. */
 string
-md5_digest (vector<boost::filesystem::path> files, shared_ptr<Job> job)
+md5_digest_head_tail (vector<boost::filesystem::path> files, boost::uintmax_t size)
 {
-	boost::uintmax_t const buffer_size = 64 * 1024;
-	char buffer[buffer_size];
-
+	boost::scoped_array<char> buffer (new char[size]);
 	MD5Digester digester;
 
-	vector<int64_t> sizes;
-	for (size_t i = 0; i < files.size(); ++i) {
-		sizes.push_back (boost::filesystem::file_size (files[i]));
-	}
-
-	for (size_t i = 0; i < files.size(); ++i) {
+	/* Head */
+	boost::uintmax_t to_do = size;
+	char* p = buffer.get ();
+	int i = 0;
+	while (i < int64_t (files.size()) && to_do > 0) {
 		FILE* f = fopen_boost (files[i], "rb");
 		if (!f) {
 			throw OpenFileError (files[i].string());
 		}
 
-		boost::uintmax_t const bytes = boost::filesystem::file_size (files[i]);
-		boost::uintmax_t remaining = bytes;
-
-		while (remaining > 0) {
-			int const t = min (remaining, buffer_size);
-			int const r = fread (buffer, 1, t, f);
-			if (r != t) {
-				throw ReadFileError (files[i], errno);
-			}
-			digester.add (buffer, t);
-			remaining -= t;
-
-			if (job) {
-				job->set_progress ((float (i) + 1 - float(remaining) / bytes) / files.size ());
-			}
+		boost::uintmax_t this_time = min (to_do, boost::filesystem::file_size (files[i]));
+		fread (p, 1, this_time, f);
+		p += this_time;
+ 		to_do -= this_time;
+		fclose (f);
+
+		++i;
+	}
+	digester.add (buffer.get(), size - to_do);
+
+	/* Tail */
+	to_do = size;
+	p = buffer.get ();
+	i = files.size() - 1;
+	while (i >= 0 && to_do > 0) {
+		FILE* f = fopen_boost (files[i], "rb");
+		if (!f) {
+			throw OpenFileError (files[i].string());
 		}
 
+		boost::uintmax_t this_time = min (to_do, boost::filesystem::file_size (files[i]));
+		fseek (f, -this_time, SEEK_END);
+		fread (p, 1, this_time, f);
+		p += this_time;
+		to_do -= this_time;
 		fclose (f);
-	}
+
+		--i;
+	}		
+	digester.add (buffer.get(), size - to_do);
 
 	return digester.get ();
 }
diff --git a/src/lib/util.h b/src/lib/util.h
index b06c8a58b..ee2865e76 100644
--- a/src/lib/util.h
+++ b/src/lib/util.h
@@ -61,7 +61,7 @@ extern std::string dependency_version_summary ();
 extern double seconds (struct timeval);
 extern void dcpomatic_setup ();
 extern void dcpomatic_setup_gettext_i18n (std::string);
-extern std::string md5_digest (std::vector<boost::filesystem::path>, boost::shared_ptr<Job>);
+extern std::string md5_digest_head_tail (std::vector<boost::filesystem::path>, boost::uintmax_t size);
 extern void ensure_ui_thread ();
 extern std::string audio_channel_name (int);
 extern bool valid_image_file (boost::filesystem::path);