/*
- Copyright (C) 2014-2015 Carl Hetherington <cth@carlh.net>
+ Copyright (C) 2014-2021 Carl Hetherington <cth@carlh.net>
This file is part of DCP-o-matic.
*/
-#include "scoped_temporary.h"
+
#include "compose.hpp"
-#include "safe_stringstream.h"
+#include "cross.h"
#include "exceptions.h"
+#include "scoped_temporary.h"
+#include "util.h"
#include <curl/curl.h>
#include <zip.h>
-#include <boost/function.hpp>
#include <boost/optional.hpp>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include "i18n.h"
-using std::string;
+
+using std::function;
using std::list;
-using boost::optional;
-using boost::function;
+using std::string;
using boost::algorithm::trim;
+using boost::optional;
+
static size_t
-get_from_zip_url_data (void* buffer, size_t size, size_t nmemb, void* stream)
+ls_url_data (void* buffer, size_t size, size_t nmemb, void* output)
+{
+ string* s = reinterpret_cast<string*>(output);
+ char* c = reinterpret_cast<char*>(buffer);
+ for (size_t i = 0; i < (size * nmemb); ++i) {
+ *s += c[i];
+ }
+ return nmemb;
+}
+
+
+list<string>
+ls_url (string url)
{
- FILE* f = reinterpret_cast<FILE*> (stream);
+ auto curl = curl_easy_init ();
+ curl_easy_setopt (curl, CURLOPT_URL, url.c_str());
+ curl_easy_setopt (curl, CURLOPT_DIRLISTONLY, 1);
+
+ string ls;
+ curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ls_url_data);
+ curl_easy_setopt (curl, CURLOPT_WRITEDATA, &ls);
+ auto const cr = curl_easy_perform (curl);
+
+ if (cr != CURLE_OK) {
+ return {};
+ }
+
+ list<string> result;
+ result.push_back("");
+ for (size_t i = 0; i < ls.size(); ++i) {
+ if (ls[i] == '\n') {
+ result.push_back("");
+ } else {
+ result.back() += ls[i];
+ }
+ }
+
+ result.pop_back ();
+ return result;
+}
+
+
+static size_t
+get_from_url_data (void* buffer, size_t size, size_t nmemb, void* stream)
+{
+ auto f = reinterpret_cast<FILE*> (stream);
return fwrite (buffer, size, nmemb, f);
}
-/** @param url URL of ZIP file.
- * @param file Filename within ZIP file.
- * @param load Function passed a (temporary) filesystem path of the unpacked file.
- */
+
optional<string>
-get_from_zip_url (string url, string file, bool pasv, function<void (boost::filesystem::path)> load)
+get_from_url (string url, bool pasv, bool skip_pasv_ip, ScopedTemporary& temp)
{
- /* Download the ZIP file to temp_zip */
- CURL* curl = curl_easy_init ();
- curl_easy_setopt (curl, CURLOPT_URL, url.c_str ());
+ auto curl = curl_easy_init ();
+ curl_easy_setopt (curl, CURLOPT_URL, url.c_str());
- ScopedTemporary temp_zip;
- FILE* f = temp_zip.open ("wb");
- curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, get_from_zip_url_data);
+ auto f = temp.open ("wb");
+ curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, get_from_url_data);
curl_easy_setopt (curl, CURLOPT_WRITEDATA, f);
curl_easy_setopt (curl, CURLOPT_FTP_USE_EPSV, 0);
curl_easy_setopt (curl, CURLOPT_FTP_USE_EPRT, 0);
+ if (skip_pasv_ip) {
+ curl_easy_setopt (curl, CURLOPT_FTP_SKIP_PASV_IP, 1);
+ }
if (!pasv) {
curl_easy_setopt (curl, CURLOPT_FTPPORT, "-");
}
/* Maximum time is 20s */
curl_easy_setopt (curl, CURLOPT_TIMEOUT, 20);
- CURLcode const cr = curl_easy_perform (curl);
+ auto const cr = curl_easy_perform (curl);
- temp_zip.close ();
+ temp.close ();
curl_easy_cleanup (curl);
if (cr != CURLE_OK) {
- return String::compose (_("Download failed (%1/%2 error %3)"), url, file, cr);
+ return String::compose (_("Download failed (%1 error %2)"), url, (int) cr);
}
- /* Open the ZIP file and read `file' out of it */
-
- struct zip* zip = zip_open (temp_zip.c_str(), 0, 0);
- if (!zip) {
- return optional<string> (_("Could not open downloaded ZIP file"));
- }
+ return {};
+}
- struct zip_file* zip_file = zip_fopen (zip, file.c_str(), 0);
- if (!zip_file) {
- return optional<string> (_("Unexpected ZIP file contents"));
- }
- ScopedTemporary temp_cert;
- f = temp_cert.open ("wb");
- char buffer[4096];
- while (true) {
- int const N = zip_fread (zip_file, buffer, sizeof (buffer));
- fwrite (buffer, 1, N, f);
- if (N < int (sizeof (buffer))) {
- break;
- }
+optional<string>
+get_from_url (string url, bool pasv, bool skip_pasv_ip, function<optional<string> (boost::filesystem::path, string)> load)
+{
+ ScopedTemporary temp;
+ auto e = get_from_url (url, pasv, skip_pasv_ip, temp);
+ if (e) {
+ return e;
}
- temp_cert.close ();
-
- load (temp_cert.file ());
- return optional<string> ();
+ return load (temp.file(), url);
}
-static size_t
-ftp_ls_data (void* buffer, size_t size, size_t nmemb, void* data)
+
+/** @param url URL of ZIP file.
+ * @param file Filename within ZIP file.
+ * @param load Function passed a (temporary) filesystem path of the unpacked file.
+ */
+optional<string>
+get_from_zip_url (string url, string file, bool pasv, bool skip_pasv_ip, function<optional<string> (boost::filesystem::path, string)> load)
{
- string* s = reinterpret_cast<string *> (data);
- uint8_t* b = reinterpret_cast<uint8_t *> (buffer);
- for (size_t i = 0; i < (size * nmemb); ++i) {
- *s += b[i];
+ /* Download the ZIP file to temp_zip */
+ ScopedTemporary temp_zip;
+ auto e = get_from_url (url, pasv, skip_pasv_ip, temp_zip);
+ if (e) {
+ return e;
}
- return size * nmemb;
-}
-list<string>
-ftp_ls (string url, bool pasv)
-{
- CURL* curl = curl_easy_init ();
- if (!curl) {
- throw NetworkError ("could not set up curl");
+ /* Open the ZIP file and read `file' out of it */
+
+#ifdef DCPOMATIC_HAVE_ZIP_SOURCE_T
+ /* This is the way to do it with newer versions of libzip, and is required on Windows.
+ The zip_source_t API is missing in the libzip versions shipped with Ubuntu 14.04,
+ Centos 6, Centos 7, Debian 7 and Debian 8.
+ */
+
+ auto zip_file = fopen_boost (temp_zip.file (), "rb");
+ if (!zip_file) {
+ return optional<string> (_("Could not open downloaded ZIP file"));
}
- if (url.substr (url.length() - 1, 1) != "/") {
- url += "/";
+ auto zip_source = zip_source_filep_create (zip_file, 0, -1, 0);
+ if (!zip_source) {
+ return optional<string> (_("Could not open downloaded ZIP file"));
}
- curl_easy_setopt (curl, CURLOPT_URL, url.c_str ());
- /* 20s timeout */
- curl_easy_setopt (curl, CURLOPT_TIMEOUT, 20);
- string ls_raw;
- struct curl_slist* commands = 0;
- commands = curl_slist_append (commands, "NLST");
- curl_easy_setopt (curl, CURLOPT_POSTQUOTE, commands);
- curl_easy_setopt (curl, CURLOPT_WRITEDATA, &ls_raw);
- curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ftp_ls_data);
- curl_easy_setopt (curl, CURLOPT_FTP_USE_EPSV, 0);
- curl_easy_setopt (curl, CURLOPT_FTP_USE_EPRT, 0);
- curl_easy_setopt (curl, CURLOPT_VERBOSE, 1);
- if (!pasv) {
- curl_easy_setopt (curl, CURLOPT_FTPPORT, "-");
+ zip_error_t error;
+ zip_error_init (&error);
+ auto zip = zip_open_from_source (zip_source, ZIP_RDONLY, &error);
+ if (!zip) {
+ return String::compose (_("Could not open downloaded ZIP file (%1:%2: %3)"), error.zip_err, error.sys_err, error.str ? error.str : "");
}
- CURLcode const r = curl_easy_perform (curl);
- if (r != CURLE_OK) {
- curl_easy_cleanup (curl);
- throw NetworkError (curl_easy_strerror (r));
+
+#else
+ struct zip* zip = zip_open (temp_zip.c_str(), 0, 0);
+#endif
+
+ struct zip_file* file_in_zip = zip_fopen (zip, file.c_str(), 0);
+ if (!file_in_zip) {
+ return optional<string> (_("Unexpected ZIP file contents"));
}
- SafeStringStream s (ls_raw);
- list<string> ls;
- while (s.good ()) {
- string line = s.getline ();
- trim (line);
- if (line.length() > 55) {
- string const file = line.substr (55);
- if (file != "." && file != "..") {
- ls.push_back (file);
- }
+ ScopedTemporary temp_cert;
+ auto f = temp_cert.open ("wb");
+ char buffer[4096];
+ while (true) {
+ int const N = zip_fread (file_in_zip, buffer, sizeof (buffer));
+ checked_fwrite (buffer, N, f, temp_cert.file());
+ if (N < int (sizeof (buffer))) {
+ break;
}
}
+ zip_fclose (file_in_zip);
+ zip_close (zip);
+ temp_cert.close ();
- curl_easy_cleanup (curl);
-
- return ls;
+ return load (temp_cert.file(), url);
}