From b00389305af83dfd1e3b41ab3b108cb591f18c5d Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Fri, 17 Jul 2015 13:52:29 +0100 Subject: [PATCH] Make a generic base for uploaders and move the SCP code into a subclass of that. --- src/lib/job.cc | 2 +- src/lib/scp_uploader.cc | 129 +++++++++++++++++++++++++++++++++++ src/lib/scp_uploader.h | 36 ++++++++++ src/lib/upload_job.cc | 144 ++-------------------------------------- src/lib/uploader.cc | 92 +++++++++++++++++++++++++ src/lib/uploader.h | 44 ++++++++++++ src/lib/wscript | 2 + test/uploader_test.cc | 83 +++++++++++++++++++++++ test/wscript | 1 + 9 files changed, 392 insertions(+), 141 deletions(-) create mode 100644 src/lib/scp_uploader.cc create mode 100644 src/lib/scp_uploader.h create mode 100644 src/lib/uploader.cc create mode 100644 src/lib/uploader.h create mode 100644 test/uploader_test.cc diff --git a/src/lib/job.cc b/src/lib/job.cc index 71e4c3446..8c46b4962 100644 --- a/src/lib/job.cc +++ b/src/lib/job.cc @@ -330,7 +330,7 @@ Job::status () const } else if (finished_ok ()) { s << String::compose (_("OK (ran for %1)"), seconds_to_hms (_ran_for)); } else if (finished_in_error ()) { - s << String::compose (_("Error (%1)"), error_summary ()); + s << String::compose (_("Error: %1"), error_summary ()); } else if (finished_cancelled ()) { s << _("Cancelled"); } diff --git a/src/lib/scp_uploader.cc b/src/lib/scp_uploader.cc new file mode 100644 index 000000000..7fb716813 --- /dev/null +++ b/src/lib/scp_uploader.cc @@ -0,0 +1,129 @@ +/* + Copyright (C) 2012-2015 Carl Hetherington + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + +#include "scp_uploader.h" +#include "exceptions.h" +#include "job.h" +#include "config.h" +#include "cross.h" +#include "compose.hpp" + +#include "i18n.h" + +using std::string; +using std::min; +using boost::shared_ptr; +using boost::function; + +SCPUploader::SCPUploader (function set_status, function set_progress) + : Uploader (set_status, set_progress) +{ + _session = ssh_new (); + if (!_session) { + throw NetworkError (_("could not start SSH session")); + } + + _set_status (_("connecting")); + + ssh_options_set (_session, SSH_OPTIONS_HOST, Config::instance()->tms_ip().c_str ()); + ssh_options_set (_session, SSH_OPTIONS_USER, Config::instance()->tms_user().c_str ()); + int const port = 22; + ssh_options_set (_session, SSH_OPTIONS_PORT, &port); + + int r = ssh_connect (_session); + if (r != SSH_OK) { + throw NetworkError (String::compose (_("Could not connect to server %1 (%2)"), Config::instance()->tms_ip(), ssh_get_error (_session))); + } + + r = ssh_is_server_known (_session); + if (r == SSH_SERVER_ERROR) { + throw NetworkError (String::compose (_("SSH error (%1)"), ssh_get_error (_session))); + } + + r = ssh_userauth_password (_session, 0, Config::instance()->tms_password().c_str ()); + if (r != SSH_AUTH_SUCCESS) { + throw NetworkError (String::compose (_("Failed to authenticate with server (%1)"), ssh_get_error (_session))); + } + + _scp = ssh_scp_new (_session, SSH_SCP_WRITE | SSH_SCP_RECURSIVE, Config::instance()->tms_path().c_str ()); + if (!_scp) { + throw NetworkError (String::compose (_("could not start SCP session (%1)"), ssh_get_error (_session))); + } + + r = ssh_scp_init (_scp); + if (r != SSH_OK) { + throw NetworkError (String::compose (_("Could not start SCP session (%1)"), ssh_get_error (_session))); + } +} + +SCPUploader::~SCPUploader () +{ + ssh_scp_free (_scp); + ssh_disconnect (_session); + ssh_free (_session); +} + +void +SCPUploader::create_directory (boost::filesystem::path directory) +{ + int const r = ssh_scp_push_directory (_scp, directory.string().c_str(), S_IRWXU); + if (r != SSH_OK) { + throw NetworkError (String::compose (_("Could not create remote directory %1 (%2)"), directory, ssh_get_error (_session))); + } +} + +void +SCPUploader::upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size) +{ + _set_status (String::compose (_("copying %1"), from.leaf ())); + + boost::uintmax_t to_do = boost::filesystem::file_size (from); + ssh_scp_push_file (_scp, to.string().c_str(), to_do, S_IRUSR | S_IWUSR); + + FILE* f = fopen_boost (from, "rb"); + if (f == 0) { + throw NetworkError (String::compose (_("Could not open %1 to send"), from)); + } + + boost::uintmax_t buffer_size = 64 * 1024; + char buffer[buffer_size]; + + while (to_do > 0) { + int const t = min (to_do, buffer_size); + size_t const read = fread (buffer, 1, t, f); + if (read != size_t (t)) { + fclose (f); + throw ReadFileError (from); + } + + int const r = ssh_scp_write (_scp, buffer, t); + if (r != SSH_OK) { + fclose (f); + throw NetworkError (String::compose (_("Could not write to remote file (%1)"), ssh_get_error (_session))); + } + to_do -= t; + transferred += t; + + if (total_size > 0) { + _set_progress ((double) transferred / total_size); + } + } + + fclose (f); +} diff --git a/src/lib/scp_uploader.h b/src/lib/scp_uploader.h new file mode 100644 index 000000000..f6cecda71 --- /dev/null +++ b/src/lib/scp_uploader.h @@ -0,0 +1,36 @@ +/* + Copyright (C) 2012-2015 Carl Hetherington + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + +#include "uploader.h" +#include + +class SCPUploader : public Uploader +{ +public: + SCPUploader (boost::function set_status, boost::function set_progress); + ~SCPUploader (); + +protected: + virtual void create_directory (boost::filesystem::path directory); + virtual void upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size); + +private: + ssh_session _session; + ssh_scp _scp; +}; diff --git a/src/lib/upload_job.cc b/src/lib/upload_job.cc index 5a0c73797..89ce5252d 100644 --- a/src/lib/upload_job.cc +++ b/src/lib/upload_job.cc @@ -34,6 +34,7 @@ #include "log.h" #include "film.h" #include "cross.h" +#include "scp_uploader.h" #include "i18n.h" @@ -43,61 +44,6 @@ using std::string; using std::min; using boost::shared_ptr; -class SSHSession -{ -public: - SSHSession () - : _connected (false) - { - session = ssh_new (); - if (session == 0) { - throw NetworkError (_("could not start SSH session")); - } - } - - int connect () - { - int r = ssh_connect (session); - if (r == 0) { - _connected = true; - } - return r; - } - - ~SSHSession () - { - if (_connected) { - ssh_disconnect (session); - } - ssh_free (session); - } - - ssh_session session; - -private: - bool _connected; -}; - -class SSHSCP -{ -public: - SSHSCP (ssh_session s) - { - scp = ssh_scp_new (s, SSH_SCP_WRITE | SSH_SCP_RECURSIVE, Config::instance()->tms_path().c_str ()); - if (!scp) { - throw NetworkError (String::compose (_("could not start SCP session (%1)"), ssh_get_error (s))); - } - } - - ~SSHSCP () - { - ssh_scp_free (scp); - } - - ssh_scp scp; -}; - - UploadJob::UploadJob (shared_ptr film) : Job (film) , _status (_("Waiting")) @@ -122,90 +68,8 @@ UploadJob::run () { LOG_GENERAL_NC (N_("Upload job starting")); - SSHSession ss; - - set_status (_("connecting")); - - ssh_options_set (ss.session, SSH_OPTIONS_HOST, Config::instance()->tms_ip().c_str ()); - ssh_options_set (ss.session, SSH_OPTIONS_USER, Config::instance()->tms_user().c_str ()); - int const port = 22; - ssh_options_set (ss.session, SSH_OPTIONS_PORT, &port); - - int r = ss.connect (); - if (r != SSH_OK) { - throw NetworkError (String::compose (_("Could not connect to server %1 (%2)"), Config::instance()->tms_ip(), ssh_get_error (ss.session))); - } - - int const state = ssh_is_server_known (ss.session); - if (state == SSH_SERVER_ERROR) { - throw NetworkError (String::compose (_("SSH error (%1)"), ssh_get_error (ss.session))); - } - - r = ssh_userauth_password (ss.session, 0, Config::instance()->tms_password().c_str ()); - if (r != SSH_AUTH_SUCCESS) { - throw NetworkError (String::compose (_("Failed to authenticate with server (%1)"), ssh_get_error (ss.session))); - } - - SSHSCP sc (ss.session); - - r = ssh_scp_init (sc.scp); - if (r != SSH_OK) { - throw NetworkError (String::compose (_("Could not start SCP session (%1)"), ssh_get_error (ss.session))); - } - - r = ssh_scp_push_directory (sc.scp, _film->dcp_name().c_str(), S_IRWXU); - if (r != SSH_OK) { - throw NetworkError (String::compose (_("Could not create remote directory %1 (%2)"), _film->dcp_name(), ssh_get_error (ss.session))); - } - - boost::filesystem::path const dcp_dir = _film->dir (_film->dcp_name()); - - boost::uintmax_t bytes_to_transfer = 0; - for (boost::filesystem::directory_iterator i = boost::filesystem::directory_iterator (dcp_dir); i != boost::filesystem::directory_iterator(); ++i) { - bytes_to_transfer += boost::filesystem::file_size (*i); - } - - boost::uintmax_t buffer_size = 64 * 1024; - char buffer[buffer_size]; - boost::uintmax_t bytes_transferred = 0; - - for (boost::filesystem::directory_iterator i = boost::filesystem::directory_iterator (dcp_dir); i != boost::filesystem::directory_iterator(); ++i) { - - string const leaf = boost::filesystem::path(*i).leaf().generic_string (); - - set_status (String::compose (_("copying %1"), leaf)); - - boost::uintmax_t to_do = boost::filesystem::file_size (*i); - ssh_scp_push_file (sc.scp, leaf.c_str(), to_do, S_IRUSR | S_IWUSR); - - FILE* f = fopen_boost (boost::filesystem::path (*i), "rb"); - if (f == 0) { - throw NetworkError (String::compose (_("Could not open %1 to send"), *i)); - } - - while (to_do > 0) { - int const t = min (to_do, buffer_size); - size_t const read = fread (buffer, 1, t, f); - if (read != size_t (t)) { - fclose (f); - throw ReadFileError (boost::filesystem::path (*i).string()); - } - - r = ssh_scp_write (sc.scp, buffer, t); - if (r != SSH_OK) { - fclose (f); - throw NetworkError (String::compose (_("Could not write to remote file (%1)"), ssh_get_error (ss.session))); - } - to_do -= t; - bytes_transferred += t; - - if (bytes_to_transfer > 0) { - set_progress ((double) bytes_transferred / bytes_to_transfer); - } - } - - fclose (f); - } + SCPUploader uploader (bind (&UploadJob::set_status, this, _1), bind (&UploadJob::set_progress, this, _1, false)); + uploader.upload (_film->dir (_film->dcp_name ())); set_progress (1); set_status (N_("")); @@ -217,7 +81,7 @@ UploadJob::status () const { boost::mutex::scoped_lock lm (_status_mutex); string s = Job::status (); - if (!_status.empty ()) { + if (!_status.empty () && !finished_in_error ()) { s += N_("; ") + _status; } return s; diff --git a/src/lib/uploader.cc b/src/lib/uploader.cc new file mode 100644 index 000000000..b6b23ed8e --- /dev/null +++ b/src/lib/uploader.cc @@ -0,0 +1,92 @@ +/* + Copyright (C) 2015 Carl Hetherington + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + +#include "uploader.h" +#include "dcpomatic_assert.h" + +using std::string; +using boost::shared_ptr; +using boost::function; + +Uploader::Uploader (function set_status, function set_progress) + : _set_status (set_status) + , _set_progress (set_progress) +{ + +} + +boost::uintmax_t +Uploader::count_file_sizes (boost::filesystem::path directory) const +{ + using namespace boost::filesystem; + + boost::uintmax_t size = 0; + + for (directory_iterator i = directory_iterator (directory); i != directory_iterator (); ++i) { + if (is_directory (i->path ())) { + size += count_file_sizes (i->path ()); + } else { + size += file_size (*i); + } + } + + return size; +} + +void +Uploader::upload (boost::filesystem::path directory) +{ + boost::uintmax_t transferred = 0; + upload_directory (directory.parent_path (), directory, transferred, count_file_sizes (directory)); +} + +void +Uploader::upload_directory (boost::filesystem::path base, boost::filesystem::path directory, boost::uintmax_t& transferred, boost::uintmax_t total_size) +{ + using namespace boost::filesystem; + + create_directory (remove_prefix (base, directory)); + for (directory_iterator i = directory_iterator (directory); i != directory_iterator (); ++i) { + if (is_directory (i->path ())) { + upload_directory (base, i->path (), transferred, total_size); + } else { + upload_file (i->path (), remove_prefix (base, i->path ()), transferred, total_size); + } + } +} + +boost::filesystem::path +Uploader::remove_prefix (boost::filesystem::path prefix, boost::filesystem::path target) const +{ + using namespace boost::filesystem; + + path result; + + path::iterator i = target.begin (); + for (path::iterator j = prefix.begin (); j != prefix.end(); ++j) { + DCPOMATIC_ASSERT (*i == *j); + ++i; + } + + for (; i != target.end(); ++i) { + result /= *i; + } + + return result; +} diff --git a/src/lib/uploader.h b/src/lib/uploader.h new file mode 100644 index 000000000..fcb9c5045 --- /dev/null +++ b/src/lib/uploader.h @@ -0,0 +1,44 @@ +/* + Copyright (C) 2015 Carl Hetherington + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + +#include +#include +#include + +class Job; + +class Uploader +{ +public: + Uploader (boost::function set_status, boost::function set_progress); + void upload (boost::filesystem::path directory); + +protected: + + virtual void create_directory (boost::filesystem::path directory) = 0; + virtual void upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t& transferred, boost::uintmax_t total_size) = 0; + + boost::function _set_status; + boost::function _set_progress; + +private: + void upload_directory (boost::filesystem::path base, boost::filesystem::path directory, boost::uintmax_t& transferred, boost::uintmax_t total_size); + boost::uintmax_t count_file_sizes (boost::filesystem::path) const; + boost::filesystem::path remove_prefix (boost::filesystem::path prefix, boost::filesystem::path target) const; +}; diff --git a/src/lib/wscript b/src/lib/wscript index 486f0557f..eb2238960 100644 --- a/src/lib/wscript +++ b/src/lib/wscript @@ -93,6 +93,7 @@ sources = """ resampler.cc safe_stringstream.cc scoped_temporary.cc + scp_uploader.cc send_kdm_email_job.cc send_problem_report_job.cc server.cc @@ -114,6 +115,7 @@ sources = """ signal_manager.cc update.cc upload_job.cc + uploader.cc upmixer_a.cc util.cc video_content.cc diff --git a/test/uploader_test.cc b/test/uploader_test.cc new file mode 100644 index 000000000..c583de667 --- /dev/null +++ b/test/uploader_test.cc @@ -0,0 +1,83 @@ +/* + Copyright (C) 2015 Carl Hetherington + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + +#include "lib/uploader.h" +#include +#include + +using std::vector; +using std::make_pair; +using std::pair; +using std::string; +using boost::shared_ptr; +using boost::bind; + +static void +set_status (string) +{ + +} + +static void +set_progress (float) +{ + +} + +class TestUploader : public Uploader +{ +public: + TestUploader () + : Uploader (bind (&set_status, _1), bind (&set_progress, _1)) + { + _directories.push_back ("uploader"); + _directories.push_back ("uploader/a"); + _directories.push_back ("uploader/c"); + _directories.push_back ("uploader/b"); + _directories.push_back ("uploader/b/e"); + _next_directory = _directories.begin (); + + _files.push_back (make_pair ("test/data/uploader/a/d", "uploader/a/d")); + _files.push_back (make_pair ("test/data/uploader/b/e/f", "uploader/b/e/f")); + _next_file = _files.begin (); + } + +protected: + void create_directory (boost::filesystem::path directory) { + BOOST_CHECK (directory == *_next_directory); + ++_next_directory; + } + + void upload_file (boost::filesystem::path from, boost::filesystem::path to, boost::uintmax_t &, boost::uintmax_t) { + BOOST_CHECK (make_pair (from, to) == *_next_file); + ++_next_file; + } + +private: + vector _directories; + vector::iterator _next_directory; + vector > _files; + vector >::iterator _next_file; +}; + +BOOST_AUTO_TEST_CASE (uploader_test) +{ + TestUploader uploader; + uploader.upload ("test/data/uploader"); +} diff --git a/test/wscript b/test/wscript index 931f3e924..bc3750904 100644 --- a/test/wscript +++ b/test/wscript @@ -80,6 +80,7 @@ def build(bld): threed_test.cc time_calculation_test.cc update_checker_test.cc + uploader_test.cc upmixer_a_test.cc util_test.cc video_content_scale_test.cc -- 2.30.2