Make a generic base for uploaders and move the SCP code into a subclass of that.

7 files changed, 308 insertions, 141 deletions

diff --git a/src/lib/job.cc b/src/lib/job.cc
index 71e4c3446..8c46b4962 100644
--- a/src/lib/job.cc
+++ b/src/lib/job.cc
@@ -330,7 +330,7 @@ Job::status () const
 	} else if (finished_ok ()) {
 		s << String::compose (_("OK (ran for %1)"), seconds_to_hms (_ran_for));
 	} else if (finished_in_error ()) {
-		s << String::compose (_("Error (%1)"), error_summary ());
+		s << String::compose (_("Error: %1"), error_summary ());
 	} else if (finished_cancelled ()) {
 		s << _("Cancelled");
 	}
diff --git a/src/lib/scp_uploader.cc b/src/lib/scp_uploader.cc
new file mode 100644
index 000000000..7fb716813
--- /dev/null
+++ b/src/lib/scp_uploader.cc
@@ -0,0 +1,129 @@
+/*
+    Copyright (C) 2012-2015 Carl Hetherington <cth@carlh.net>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+
+#include "scp_uploader.h"
+#include "exceptions.h"
+#include "job.h"
+#include "config.h"
+#include "cross.h"
+#include "compose.hpp"
+
+#include "i18n.h"
+
+using std::string;
+using std::min;
+using boost::shared_ptr;
+using boost::function;
+
+SCPUploader::SCPUploader (function<void (string)> set_status, function<void (float)> set_progress)
+	: Uploader (set_status, set_progress)
+{
+	_session = ssh_new ();
+	if (!_session) {
+		throw NetworkError (_("could not start SSH session"));
+	}
+
+	_set_status (_("connecting"));
+
+	ssh_options_set (_session, SSH_OPTIONS_HOST, Config::instance()->tms_ip().c_str ());
+	ssh_options_set (_session, SSH_OPTIONS_USER, Config::instance()->tms_user().c_str ());
+	int const port = 22;
+	ssh_options_set (_session, SSH_OPTIONS_PORT, &port);
+
+	int r = ssh_connect (_session);
+	if (r != SSH_OK) {
+		throw NetworkError (String::compose (_("Could not connect to server %1 (%2)"), Config::instance()->tms_ip(), ssh_get_error (_session)));
+	}
+
+	r = ssh_is_server_known (_session);
+	if (r == SSH_SERVER_ERROR) {
+		throw NetworkError (String::compose (_("SSH error (%1)"), ssh_get_error (_session)));
+	}
+
+	r = ssh_userauth_password (_session, 0, Config::instance()->tms_password().c_str ());
+	if (r != SSH_AUTH_SUCCESS) {
+		throw NetworkError (String::compose (_("Failed to authenticate with server (%1)"), ssh_get_error (_session)));
+	}
+
+	_scp = ssh_scp_new (_session, SSH_SCP_WRITE | SSH_SCP_RECURSIVE, Config::instance()->tms_path().c_str ());
+	if (!_scp) {
+		throw NetworkError (String::compose (_("could not start SCP session (%1)"), ssh_get_error (_session)));
+	}
+
+	r = ssh_scp_init (_scp);
+	if (r != SSH_OK) {
+		throw NetworkError (String::compose (_("Could not start SCP session (%1)"), ssh_get_error (_session)));
+	}
+}
+
+SCPUploader::~SCPUploader ()
+{
+	ssh_scp_free (_scp);
+	ssh_disconnect (_session);
+	ssh_free (_session);
+}
+
+void
+SCPUploader::create_directory (boost::filesystem::path directory)
+{
+	int const r = ssh_scp_push_directory (_scp, directory.string().c_str(), S_IRWXU);
+	if (r != SSH_OK) {
+		throw NetworkError (String::compose (_("Could not create remote directory %1 (%2)"), directory, ssh_get_error (_session)));
+	}
+}
+
+void
+SCPUploader::upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size)
+{
+	_set_status (String::compose (_("copying %1"), from.leaf ()));
+
+	boost::uintmax_t to_do = boost::filesystem::file_size (from);
+	ssh_scp_push_file (_scp, to.string().c_str(), to_do, S_IRUSR | S_IWUSR);
+
+	FILE* f = fopen_boost (from, "rb");
+	if (f == 0) {
+		throw NetworkError (String::compose (_("Could not open %1 to send"), from));
+	}
+
+	boost::uintmax_t buffer_size = 64 * 1024;
+	char buffer[buffer_size];
+
+	while (to_do > 0) {
+		int const t = min (to_do, buffer_size);
+		size_t const read = fread (buffer, 1, t, f);
+		if (read != size_t (t)) {
+			fclose (f);
+			throw ReadFileError (from);
+		}
+
+		int const r = ssh_scp_write (_scp, buffer, t);
+		if (r != SSH_OK) {
+			fclose (f);
+			throw NetworkError (String::compose (_("Could not write to remote file (%1)"), ssh_get_error (_session)));
+		}
+		to_do -= t;
+		transferred += t;
+
+		if (total_size > 0) {
+			_set_progress ((double) transferred / total_size);
+		}
+	}
+
+	fclose (f);
+}
diff --git a/src/lib/scp_uploader.h b/src/lib/scp_uploader.h
new file mode 100644
index 000000000..f6cecda71
--- /dev/null
+++ b/src/lib/scp_uploader.h
@@ -0,0 +1,36 @@
+/*
+    Copyright (C) 2012-2015 Carl Hetherington <cth@carlh.net>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+
+#include "uploader.h"
+#include <libssh/libssh.h>
+
+class SCPUploader : public Uploader
+{
+public:
+	SCPUploader (boost::function<void (std::string)> set_status, boost::function<void (float)> set_progress);
+	~SCPUploader ();
+
+protected:
+	virtual void create_directory (boost::filesystem::path directory);
+	virtual void upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size);
+
+private:
+	ssh_session _session;
+	ssh_scp _scp;
+};
diff --git a/src/lib/upload_job.cc b/src/lib/upload_job.cc
index 5a0c73797..89ce5252d 100644
--- a/src/lib/upload_job.cc
+++ b/src/lib/upload_job.cc
@@ -34,6 +34,7 @@
 #include "log.h"
 #include "film.h"
 #include "cross.h"
+#include "scp_uploader.h"
 
 #include "i18n.h"
 
@@ -43,61 +44,6 @@ using std::string;
 using std::min;
 using boost::shared_ptr;
 
-class SSHSession
-{
-public:
-	SSHSession ()
-		: _connected (false)
-	{
-		session = ssh_new ();
-		if (session == 0) {
-			throw NetworkError (_("could not start SSH session"));
-		}
-	}
-
-	int connect ()
-	{
-		int r = ssh_connect (session);
-		if (r == 0) {
-			_connected = true;
-		}
-		return r;
-	}
-
-	~SSHSession ()
-	{
-		if (_connected) {
-			ssh_disconnect (session);
-		}
-		ssh_free (session);
-	}
-
-	ssh_session session;
-
-private:
-	bool _connected;
-};
-
-class SSHSCP
-{
-public:
-	SSHSCP (ssh_session s)
-	{
-		scp = ssh_scp_new (s, SSH_SCP_WRITE | SSH_SCP_RECURSIVE, Config::instance()->tms_path().c_str ());
-		if (!scp) {
-			throw NetworkError (String::compose (_("could not start SCP session (%1)"), ssh_get_error (s)));
-		}
-	}
-
-	~SSHSCP ()
-	{
-		ssh_scp_free (scp);
-	}
-
-	ssh_scp scp;
-};
-
-
 UploadJob::UploadJob (shared_ptr<const Film> film)
 	: Job (film)
 	, _status (_("Waiting"))
@@ -122,90 +68,8 @@ UploadJob::run ()
 {
 	LOG_GENERAL_NC (N_("Upload job starting"));
 
-	SSHSession ss;
-
-	set_status (_("connecting"));
-
-	ssh_options_set (ss.session, SSH_OPTIONS_HOST, Config::instance()->tms_ip().c_str ());
-	ssh_options_set (ss.session, SSH_OPTIONS_USER, Config::instance()->tms_user().c_str ());
-	int const port = 22;
-	ssh_options_set (ss.session, SSH_OPTIONS_PORT, &port);
-
-	int r = ss.connect ();
-	if (r != SSH_OK) {
-		throw NetworkError (String::compose (_("Could not connect to server %1 (%2)"), Config::instance()->tms_ip(), ssh_get_error (ss.session)));
-	}
-
-	int const state = ssh_is_server_known (ss.session);
-	if (state == SSH_SERVER_ERROR) {
-		throw NetworkError (String::compose (_("SSH error (%1)"), ssh_get_error (ss.session)));
-	}
-
-	r = ssh_userauth_password (ss.session, 0, Config::instance()->tms_password().c_str ());
-	if (r != SSH_AUTH_SUCCESS) {
-		throw NetworkError (String::compose (_("Failed to authenticate with server (%1)"), ssh_get_error (ss.session)));
-	}
-
-	SSHSCP sc (ss.session);
-
-	r = ssh_scp_init (sc.scp);
-	if (r != SSH_OK) {
-		throw NetworkError (String::compose (_("Could not start SCP session (%1)"), ssh_get_error (ss.session)));
-	}
-
-	r = ssh_scp_push_directory (sc.scp, _film->dcp_name().c_str(), S_IRWXU);
-	if (r != SSH_OK) {
-		throw NetworkError (String::compose (_("Could not create remote directory %1 (%2)"), _film->dcp_name(), ssh_get_error (ss.session)));
-	}
-
-	boost::filesystem::path const dcp_dir = _film->dir (_film->dcp_name());
-
-	boost::uintmax_t bytes_to_transfer = 0;
-	for (boost::filesystem::directory_iterator i = boost::filesystem::directory_iterator (dcp_dir); i != boost::filesystem::directory_iterator(); ++i) {
-		bytes_to_transfer += boost::filesystem::file_size (*i);
-	}
-
-	boost::uintmax_t buffer_size = 64 * 1024;
-	char buffer[buffer_size];
-	boost::uintmax_t bytes_transferred = 0;
-
-	for (boost::filesystem::directory_iterator i = boost::filesystem::directory_iterator (dcp_dir); i != boost::filesystem::directory_iterator(); ++i) {
-
-		string const leaf = boost::filesystem::path(*i).leaf().generic_string ();
-
-		set_status (String::compose (_("copying %1"), leaf));
-
-		boost::uintmax_t to_do = boost::filesystem::file_size (*i);
-		ssh_scp_push_file (sc.scp, leaf.c_str(), to_do, S_IRUSR | S_IWUSR);
-
-		FILE* f = fopen_boost (boost::filesystem::path (*i), "rb");
-		if (f == 0) {
-			throw NetworkError (String::compose (_("Could not open %1 to send"), *i));
-		}
-
-		while (to_do > 0) {
-			int const t = min (to_do, buffer_size);
-			size_t const read = fread (buffer, 1, t, f);
-			if (read != size_t (t)) {
-				fclose (f);
-				throw ReadFileError (boost::filesystem::path (*i).string());
-			}
-
-			r = ssh_scp_write (sc.scp, buffer, t);
-			if (r != SSH_OK) {
-				fclose (f);
-				throw NetworkError (String::compose (_("Could not write to remote file (%1)"), ssh_get_error (ss.session)));
-			}
-			to_do -= t;
-			bytes_transferred += t;
-
-			if (bytes_to_transfer > 0) {
-				set_progress ((double) bytes_transferred / bytes_to_transfer);
-			}
-		}
-
-		fclose (f);
-	}
+	SCPUploader uploader (bind (&UploadJob::set_status, this, _1), bind (&UploadJob::set_progress, this, _1, false));
+	uploader.upload (_film->dir (_film->dcp_name ()));
 
 	set_progress (1);
 	set_status (N_(""));
@@ -217,7 +81,7 @@ UploadJob::status () const
 {
 	boost::mutex::scoped_lock lm (_status_mutex);
 	string s = Job::status ();
-	if (!_status.empty ()) {
+	if (!_status.empty () && !finished_in_error ()) {
 		s += N_("; ") + _status;
 	}
 	return s;
diff --git a/src/lib/uploader.cc b/src/lib/uploader.cc
new file mode 100644
index 000000000..b6b23ed8e
--- /dev/null
+++ b/src/lib/uploader.cc
@@ -0,0 +1,92 @@
+/*
+    Copyright (C) 2015 Carl Hetherington <cth@carlh.net>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+
+#include "uploader.h"
+#include "dcpomatic_assert.h"
+
+using std::string;
+using boost::shared_ptr;
+using boost::function;
+
+Uploader::Uploader (function<void (string)> set_status, function<void (float)> set_progress)
+	: _set_status (set_status)
+	, _set_progress (set_progress)
+{
+
+}
+
+boost::uintmax_t
+Uploader::count_file_sizes (boost::filesystem::path directory) const
+{
+	using namespace boost::filesystem;
+
+	boost::uintmax_t size = 0;
+
+	for (directory_iterator i = directory_iterator (directory); i != directory_iterator (); ++i) {
+		if (is_directory (i->path ())) {
+			size += count_file_sizes (i->path ());
+		} else {
+			size += file_size (*i);
+		}
+	}
+
+	return size;
+}
+
+void
+Uploader::upload (boost::filesystem::path directory)
+{
+	boost::uintmax_t transferred = 0;
+	upload_directory (directory.parent_path (), directory, transferred, count_file_sizes (directory));
+}
+
+void
+Uploader::upload_directory (boost::filesystem::path base, boost::filesystem::path directory, boost::uintmax_t& transferred, boost::uintmax_t total_size)
+{
+	using namespace boost::filesystem;
+
+	create_directory (remove_prefix (base, directory));
+	for (directory_iterator i = directory_iterator (directory); i != directory_iterator (); ++i) {
+		if (is_directory (i->path ())) {
+			upload_directory (base, i->path (), transferred, total_size);
+		} else {
+			upload_file (i->path (), remove_prefix (base, i->path ()), transferred, total_size);
+		}
+	}
+}
+
+boost::filesystem::path
+Uploader::remove_prefix (boost::filesystem::path prefix, boost::filesystem::path target) const
+{
+	using namespace boost::filesystem;
+
+	path result;
+
+	path::iterator i = target.begin ();
+	for (path::iterator j = prefix.begin (); j != prefix.end(); ++j) {
+		DCPOMATIC_ASSERT (*i == *j);
+		++i;
+	}
+
+	for (; i != target.end(); ++i) {
+		result /= *i;
+	}
+
+	return result;
+}
diff --git a/src/lib/uploader.h b/src/lib/uploader.h
new file mode 100644
index 000000000..fcb9c5045
--- /dev/null
+++ b/src/lib/uploader.h
@@ -0,0 +1,44 @@
+/*
+    Copyright (C) 2015 Carl Hetherington <cth@carlh.net>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+
+#include <boost/shared_ptr.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/function.hpp>
+
+class Job;
+
+class Uploader
+{
+public:
+	Uploader (boost::function<void (std::string)> set_status, boost::function<void (float)> set_progress);
+	void upload (boost::filesystem::path directory);
+
+protected:
+
+	virtual void create_directory (boost::filesystem::path directory) = 0;
+	virtual void upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size) = 0;
+
+	boost::function<void (std::string)> _set_status;
+	boost::function<void (float)> _set_progress;
+
+private:
+	void upload_directory (boost::filesystem::path base, boost::filesystem::path directory, boost::uintmax_t& transferred, boost::uintmax_t total_size);
+	boost::uintmax_t count_file_sizes (boost::filesystem::path) const;
+	boost::filesystem::path remove_prefix (boost::filesystem::path prefix, boost::filesystem::path target) const;
+};
diff --git a/src/lib/wscript b/src/lib/wscript
index 486f0557f..eb2238960 100644
--- a/src/lib/wscript
+++ b/src/lib/wscript
@@ -93,6 +93,7 @@ sources = """
           resampler.cc
           safe_stringstream.cc
           scoped_temporary.cc
+          scp_uploader.cc
           send_kdm_email_job.cc
           send_problem_report_job.cc
           server.cc
@@ -114,6 +115,7 @@ sources = """
           signal_manager.cc
           update.cc
           upload_job.cc
+          uploader.cc
           upmixer_a.cc
           util.cc
           video_content.cc