mirror of
https://github.com/Sude-/lgogdownloader.git
synced 2025-02-02 05:52:31 +01:00
Add retry support to --galaxy-install
Added retry support to --galaxy-install Changed some retry conditions Show reason for retry attempts Added some Curl related helper functions to Util
This commit is contained in:
Sude
parent
17bebfc374
commit
26315b23b8
@ -27,6 +27,7 @@
|
||||
#include "threadsafequeue.h"
|
||||
#include "galaxyapi.h"
|
||||
#include "globals.h"
|
||||
#include "util.h"
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <json/json.h>
|
||||
@ -60,12 +61,6 @@ struct xferInfo
|
||||
curl_off_t offset;
|
||||
};
|
||||
|
||||
struct ChunkMemoryStruct
|
||||
{
|
||||
char *memory;
|
||||
curl_off_t size;
|
||||
};
|
||||
|
||||
typedef struct
|
||||
{
|
||||
std::string filepath;
|
||||
@ -142,7 +137,6 @@ class Downloader
|
||||
static void getGameDetailsThread(Config config, const unsigned int& tid);
|
||||
|
||||
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
|
||||
static size_t writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp);
|
||||
static size_t writeData(void *ptr, size_t size, size_t nmemb, FILE *stream);
|
||||
static size_t readData(void *ptr, size_t size, size_t nmemb, FILE *stream);
|
||||
|
||||
|
||||
@ -23,6 +23,13 @@
|
||||
#include <boost/regex.hpp>
|
||||
#include <json/json.h>
|
||||
#include <boost/date_time/posix_time/posix_time.hpp>
|
||||
#include <curl/curl.h>
|
||||
|
||||
typedef struct
|
||||
{
|
||||
char *memory;
|
||||
curl_off_t size;
|
||||
} ChunkMemoryStruct;
|
||||
|
||||
struct gameItem
|
||||
{
|
||||
@ -77,6 +84,11 @@ namespace Util
|
||||
std::string getStrippedString(std::string str);
|
||||
std::string makeEtaString(const unsigned long long& iBytesRemaining, const double& dlRate);
|
||||
std::string makeEtaString(const boost::posix_time::time_duration& duration);
|
||||
void CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf);
|
||||
CURLcode CurlGetResponse(const std::string& url, std::string& response, int max_retries = -1);
|
||||
CURLcode CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries = -1);
|
||||
curl_off_t CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp);
|
||||
curl_off_t CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp);
|
||||
|
||||
template<typename ... Args> std::string formattedString(const std::string& format, Args ... args)
|
||||
{
|
||||
|
||||
@ -27,7 +27,6 @@ class Website
|
||||
virtual ~Website();
|
||||
protected:
|
||||
private:
|
||||
static size_t writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp);
|
||||
CURL* curlhandle;
|
||||
bool IsloggedInSimple();
|
||||
bool IsLoggedInComplex(const std::string& email);
|
||||
|
||||
@ -50,25 +50,6 @@ ThreadSafeQueue<zipFileEntry> dlQueueGalaxy_MojoSetupHack;
|
||||
std::mutex mtx_create_directories; // Mutex for creating directories in Downloader::processDownloadQueue
|
||||
std::atomic<unsigned long long> iTotalRemainingBytes(0);
|
||||
|
||||
static curl_off_t WriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp)
|
||||
{
|
||||
curl_off_t realsize = size * nmemb;
|
||||
struct ChunkMemoryStruct *mem = (struct ChunkMemoryStruct *)userp;
|
||||
|
||||
mem->memory = (char *) realloc(mem->memory, mem->size + realsize + 1);
|
||||
if(mem->memory == NULL)
|
||||
{
|
||||
std::cout << "Not enough memory (realloc returned NULL)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
memcpy(&(mem->memory[mem->size]), contents, realsize);
|
||||
mem->size += realsize;
|
||||
mem->memory[mem->size] = 0;
|
||||
|
||||
return realsize;
|
||||
}
|
||||
|
||||
Downloader::Downloader()
|
||||
{
|
||||
if (Globals::globalConfig.bLogin)
|
||||
@ -86,26 +67,13 @@ Downloader::Downloader()
|
||||
|
||||
// Initialize curl and set curl options
|
||||
curlhandle = curl_easy_init();
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, Globals::globalConfig.curlConf.sUserAgent.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, Globals::globalConfig.curlConf.iTimeout);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, Globals::globalConfig.curlConf.bVerifyPeer);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, Globals::globalConfig.curlConf.bVerbose);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, Globals::globalConfig.curlConf.iDownloadRate);
|
||||
Util::CurlHandleSetDefaultOptions(curlhandle, Globals::globalConfig.curlConf);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_XFERINFODATA, this);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||
|
||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, Globals::globalConfig.curlConf.iLowSpeedTimeout);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, Globals::globalConfig.curlConf.iLowSpeedTimeoutRate);
|
||||
|
||||
if (!Globals::globalConfig.curlConf.sCACertPath.empty())
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, Globals::globalConfig.curlConf.sCACertPath.c_str());
|
||||
|
||||
// Create new GOG website handle
|
||||
gogWebsite = new Website();
|
||||
@ -1513,21 +1481,9 @@ std::string Downloader::getResponse(const std::string& url)
|
||||
std::string response;
|
||||
|
||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||
|
||||
CURLcode result;
|
||||
do
|
||||
{
|
||||
if (Globals::globalConfig.iWait > 0)
|
||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
||||
result = curl_easy_perform(curlhandle);
|
||||
response = memory.str();
|
||||
memory.str(std::string());
|
||||
}
|
||||
while ((result != CURLE_OK) && response.empty() && (this->retries++ < Globals::globalConfig.iRetries));
|
||||
this->retries = 0; // reset retries counter
|
||||
int max_retries = std::min(3, Globals::globalConfig.iRetries);
|
||||
CURLcode result = Util::CurlHandleGetResponse(curlhandle, response, max_retries);
|
||||
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0);
|
||||
@ -1645,13 +1601,6 @@ int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t d
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t Downloader::writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp) {
|
||||
std::ostringstream *stream = (std::ostringstream*)userp;
|
||||
size_t count = size * nmemb;
|
||||
stream->write(ptr, count);
|
||||
return count;
|
||||
}
|
||||
|
||||
size_t Downloader::writeData(void *ptr, size_t size, size_t nmemb, FILE *stream)
|
||||
{
|
||||
return fwrite(ptr, size, nmemb, stream);
|
||||
@ -2552,27 +2501,12 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
||||
}
|
||||
|
||||
CURL* dlhandle = curl_easy_init();
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.curlConf.sUserAgent.c_str());
|
||||
Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOSIGNAL, 1);
|
||||
|
||||
curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.curlConf.iTimeout);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_SSL_VERIFYPEER, conf.curlConf.bVerifyPeer);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_VERBOSE, conf.curlConf.bVerbose);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.curlConf.iDownloadRate);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||
|
||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_TIME, conf.curlConf.iLowSpeedTimeout);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.curlConf.iLowSpeedTimeoutRate);
|
||||
|
||||
if (!conf.curlConf.sCACertPath.empty())
|
||||
curl_easy_setopt(dlhandle, CURLOPT_CAINFO, conf.curlConf.sCACertPath.c_str());
|
||||
|
||||
xferInfo xferinfo;
|
||||
xferinfo.tid = tid;
|
||||
xferinfo.curlhandle = dlhandle;
|
||||
@ -2804,6 +2738,7 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
||||
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
|
||||
long int response_code = 0;
|
||||
bool bShouldRetry = false;
|
||||
std::string retry_reason;
|
||||
do
|
||||
{
|
||||
if (conf.iWait > 0)
|
||||
@ -2812,7 +2747,13 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
||||
response_code = 0; // Make sure that response code is reset
|
||||
|
||||
if (iRetryCount != 0)
|
||||
msgQueue.push(Message("Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix));
|
||||
{
|
||||
std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string();
|
||||
if (!retry_reason.empty())
|
||||
retry_msg += " (" + retry_reason + ")";
|
||||
msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix));
|
||||
}
|
||||
retry_reason = ""; // reset retry reason
|
||||
|
||||
FILE* outfile;
|
||||
// File exists, resume
|
||||
@ -2875,6 +2816,7 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
||||
if (bShouldRetry)
|
||||
{
|
||||
iRetryCount++;
|
||||
retry_reason = std::string(curl_easy_strerror(result));
|
||||
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
|
||||
bResume = true;
|
||||
}
|
||||
@ -3649,27 +3591,12 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
||||
}
|
||||
|
||||
CURL* dlhandle = curl_easy_init();
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.curlConf.sUserAgent.c_str());
|
||||
Util::CurlHandleSetDefaultOptions(dlhandle, Globals::globalConfig.curlConf);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOSIGNAL, 1);
|
||||
|
||||
curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.curlConf.iTimeout);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_SSL_VERIFYPEER, conf.curlConf.bVerifyPeer);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_VERBOSE, conf.curlConf.bVerbose);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.curlConf.iDownloadRate);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||
|
||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_TIME, conf.curlConf.iLowSpeedTimeout);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.curlConf.iLowSpeedTimeoutRate);
|
||||
|
||||
if (!conf.curlConf.sCACertPath.empty())
|
||||
curl_easy_setopt(dlhandle, CURLOPT_CAINFO, conf.curlConf.sCACertPath.c_str());
|
||||
|
||||
xferInfo xferinfo;
|
||||
xferinfo.tid = tid;
|
||||
xferinfo.curlhandle = dlhandle;
|
||||
@ -3829,10 +3756,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
||||
std::time_t timestamp = -1;
|
||||
for (unsigned int j = start_chunk; j < item.chunks.size(); ++j)
|
||||
{
|
||||
ChunkMemoryStruct chunk;
|
||||
chunk.memory = (char *) malloc(1);
|
||||
chunk.size = 0;
|
||||
|
||||
// Refresh Galaxy login if token is expired
|
||||
if (galaxy->isTokenExpired())
|
||||
{
|
||||
@ -3840,7 +3763,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
||||
{
|
||||
msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix));
|
||||
vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
|
||||
free(chunk.memory);
|
||||
delete galaxy;
|
||||
return;
|
||||
}
|
||||
@ -3857,7 +3779,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
||||
bChunkFailure = true;
|
||||
std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")";
|
||||
msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix));
|
||||
free(chunk.memory);
|
||||
break;
|
||||
}
|
||||
|
||||
@ -3925,7 +3846,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
||||
{
|
||||
bChunkFailure = true;
|
||||
msgQueue.push(Message(path.string() + ": Failed to get download url", MSGTYPE_ERROR, msg_prefix));
|
||||
free(chunk.memory);
|
||||
break;
|
||||
}
|
||||
|
||||
@ -3940,25 +3860,78 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
||||
// Select url with lowest priority score
|
||||
std::string url = cdnUrls[0].url;
|
||||
|
||||
ChunkMemoryStruct chunk;
|
||||
chunk.memory = (char *) malloc(1);
|
||||
chunk.size = 0;
|
||||
|
||||
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, WriteChunkMemoryCallback);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteChunkMemoryCallback);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &chunk);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, 0);
|
||||
|
||||
std::string filepath_and_chunk = path.string() + " (chunk " + std::to_string(j + 1) + "/" + std::to_string(item.chunks.size()) + ")";
|
||||
vDownloadInfo[tid].setFilename(filepath_and_chunk);
|
||||
|
||||
CURLcode result;
|
||||
int iRetryCount = 0;
|
||||
long int response_code = 0;
|
||||
bool bShouldRetry = false;
|
||||
std::string retry_reason;
|
||||
do
|
||||
{
|
||||
if (Globals::globalConfig.iWait > 0)
|
||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
||||
|
||||
xferinfo.offset = 0;
|
||||
response_code = 0; // Make sure that response code is reset
|
||||
|
||||
if (iRetryCount != 0)
|
||||
{
|
||||
std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath_and_chunk;
|
||||
if (!retry_reason.empty())
|
||||
retry_msg += " (" + retry_reason + ")";
|
||||
msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix));
|
||||
|
||||
curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, chunk.size);
|
||||
}
|
||||
retry_reason = ""; // reset retry reason
|
||||
|
||||
xferinfo.offset = chunk.size;
|
||||
xferinfo.timer.reset();
|
||||
xferinfo.TimeAndSize.clear();
|
||||
result = curl_easy_perform(dlhandle);
|
||||
|
||||
CURLcode result = curl_easy_perform(dlhandle);
|
||||
switch (result)
|
||||
{
|
||||
// Retry on these errors
|
||||
case CURLE_PARTIAL_FILE:
|
||||
case CURLE_OPERATION_TIMEDOUT:
|
||||
case CURLE_RECV_ERROR:
|
||||
bShouldRetry = true;
|
||||
break;
|
||||
// Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable"
|
||||
case CURLE_HTTP_RETURNED_ERROR:
|
||||
curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
if (response_code == 416)
|
||||
bShouldRetry = false;
|
||||
else
|
||||
bShouldRetry = true;
|
||||
break;
|
||||
default:
|
||||
bShouldRetry = false;
|
||||
break;
|
||||
}
|
||||
|
||||
if (bShouldRetry)
|
||||
{
|
||||
iRetryCount++;
|
||||
retry_reason = std::string(curl_easy_strerror(result));
|
||||
}
|
||||
|
||||
} while (bShouldRetry && (iRetryCount <= conf.iRetries));
|
||||
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||
@ -4277,7 +4250,7 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
|
||||
std::stringstream splitfiles_uncompressed;
|
||||
|
||||
CURLcode result = CURLE_RECV_ERROR;
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &splitfiles_compressed);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, dlrange.c_str());
|
||||
result = curl_easy_perform(curlhandle);
|
||||
@ -4581,27 +4554,12 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
|
||||
std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
|
||||
|
||||
CURL* dlhandle = curl_easy_init();
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.curlConf.sUserAgent.c_str());
|
||||
Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_NOSIGNAL, 1);
|
||||
|
||||
curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.curlConf.iTimeout);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_SSL_VERIFYPEER, conf.curlConf.bVerifyPeer);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_VERBOSE, conf.curlConf.bVerbose);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.curlConf.iDownloadRate);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||
|
||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_TIME, conf.curlConf.iLowSpeedTimeout);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.curlConf.iLowSpeedTimeoutRate);
|
||||
|
||||
if (!conf.curlConf.sCACertPath.empty())
|
||||
curl_easy_setopt(dlhandle, CURLOPT_CAINFO, conf.curlConf.sCACertPath.c_str());
|
||||
|
||||
xferInfo xferinfo;
|
||||
xferinfo.tid = tid;
|
||||
xferinfo.curlhandle = dlhandle;
|
||||
@ -4743,7 +4701,7 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
|
||||
std::string link_target;
|
||||
|
||||
CURLcode result = CURLE_RECV_ERROR;
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &symlink_compressed);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str());
|
||||
|
||||
@ -4826,7 +4784,7 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
|
||||
|
||||
std::stringstream data_compressed;
|
||||
vDownloadInfo[tid].setFilename(path.string());
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &data_compressed);
|
||||
curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str());
|
||||
|
||||
@ -5111,7 +5069,7 @@ int Downloader::mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEn
|
||||
std::stringstream head;
|
||||
curl_easy_setopt(curlhandle, CURLOPT_URL, installer_url.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &head);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, head_range.c_str());
|
||||
result = curl_easy_perform(curlhandle);
|
||||
@ -5142,7 +5100,7 @@ int Downloader::mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEn
|
||||
// Get tail
|
||||
std::stringstream tail;
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str());
|
||||
result = curl_easy_perform(curlhandle);
|
||||
@ -5186,7 +5144,7 @@ int Downloader::mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEn
|
||||
tail.str(std::string());
|
||||
tail_range = std::to_string(mojosetup_cd_offset) + "-" + std::to_string(file_size);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str());
|
||||
result = curl_easy_perform(curlhandle);
|
||||
|
||||
@ -26,25 +26,7 @@ galaxyAPI::galaxyAPI(CurlConfig& conf)
|
||||
this->curlConf = conf;
|
||||
|
||||
curlhandle = curl_easy_init();
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_PROGRESSDATA, this);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, curlConf.iTimeout);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, curlConf.sCookiePath.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, curlConf.sCookiePath.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, curlConf.bVerifyPeer);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, curlConf.bVerbose);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, curlConf.iDownloadRate);
|
||||
|
||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, curlConf.iLowSpeedTimeout);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, curlConf.iLowSpeedTimeoutRate);
|
||||
|
||||
if (!curlConf.sCACertPath.empty())
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, curlConf.sCACertPath.c_str());
|
||||
Util::CurlHandleSetDefaultOptions(curlhandle, this->curlConf);
|
||||
}
|
||||
|
||||
galaxyAPI::~galaxyAPI()
|
||||
@ -99,8 +81,6 @@ bool galaxyAPI::isTokenExpired()
|
||||
|
||||
std::string galaxyAPI::getResponse(const std::string& url, const bool& zlib_decompress)
|
||||
{
|
||||
std::ostringstream memory;
|
||||
|
||||
struct curl_slist *header = NULL;
|
||||
|
||||
std::string access_token;
|
||||
@ -112,24 +92,11 @@ std::string galaxyAPI::getResponse(const std::string& url, const bool& zlib_deco
|
||||
header = curl_slist_append(header, bearer.c_str());
|
||||
}
|
||||
curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, header);
|
||||
|
||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, galaxyAPI::writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||
|
||||
int retries = 0;
|
||||
CURLcode result;
|
||||
int max_retries = std::min(3, Globals::globalConfig.iRetries);
|
||||
std::string response;
|
||||
do
|
||||
{
|
||||
if (Globals::globalConfig.iWait > 0)
|
||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
||||
result = curl_easy_perform(curlhandle);
|
||||
response = memory.str();
|
||||
memory.str(std::string());
|
||||
}
|
||||
while ((result != CURLE_OK) && response.empty() && (retries++ < Globals::globalConfig.iRetries));
|
||||
Util::CurlHandleGetResponse(curlhandle, response, max_retries);
|
||||
|
||||
curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, NULL);
|
||||
curl_slist_free_all(header);
|
||||
|
||||
116
src/util.cpp
116
src/util.cpp
@ -732,3 +732,119 @@ std::string Util::makeEtaString(const boost::posix_time::time_duration& duration
|
||||
|
||||
return etastr;
|
||||
}
|
||||
|
||||
void Util::CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf)
|
||||
{
|
||||
curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, conf.sUserAgent.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, conf.iTimeout);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, conf.sCookiePath.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, conf.sCookiePath.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, conf.bVerifyPeer);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, conf.bVerbose);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.iDownloadRate);
|
||||
|
||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, conf.iLowSpeedTimeout);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.iLowSpeedTimeoutRate);
|
||||
|
||||
if (!conf.sCACertPath.empty())
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, conf.sCACertPath.c_str());
|
||||
}
|
||||
|
||||
CURLcode Util::CurlGetResponse(const std::string& url, std::string& response, int max_retries)
|
||||
{
|
||||
CURLcode result;
|
||||
CURL *handle = curl_easy_init();
|
||||
Util::CurlHandleSetDefaultOptions(handle, Globals::globalConfig.curlConf);
|
||||
curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
|
||||
|
||||
result = Util::CurlHandleGetResponse(handle, response, max_retries);
|
||||
|
||||
curl_easy_cleanup(handle);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
CURLcode Util::CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries)
|
||||
{
|
||||
CURLcode result;
|
||||
int retries = 0;
|
||||
std::ostringstream memory;
|
||||
bool bShouldRetry = false;
|
||||
long int response_code = 0;
|
||||
if (max_retries < 0)
|
||||
max_retries = Globals::globalConfig.iRetries;
|
||||
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||
|
||||
do
|
||||
{
|
||||
if (bShouldRetry)
|
||||
retries++;
|
||||
|
||||
if (Globals::globalConfig.iWait > 0)
|
||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
||||
result = curl_easy_perform(curlhandle);
|
||||
response = memory.str();
|
||||
memory.str(std::string());
|
||||
|
||||
switch (result)
|
||||
{
|
||||
// Retry on these errors
|
||||
case CURLE_PARTIAL_FILE:
|
||||
case CURLE_OPERATION_TIMEDOUT:
|
||||
case CURLE_RECV_ERROR:
|
||||
bShouldRetry = true;
|
||||
break;
|
||||
// Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "404 Not Found"
|
||||
case CURLE_HTTP_RETURNED_ERROR:
|
||||
curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
if (response_code == 404)
|
||||
bShouldRetry = false;
|
||||
else
|
||||
bShouldRetry = true;
|
||||
break;
|
||||
default:
|
||||
bShouldRetry = false;
|
||||
break;
|
||||
}
|
||||
if (retries >= max_retries)
|
||||
bShouldRetry = false;
|
||||
} while (bShouldRetry);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
curl_off_t Util::CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp)
|
||||
{
|
||||
std::ostringstream *stream = (std::ostringstream*)userp;
|
||||
std::streamsize count = (std::streamsize) size * nmemb;
|
||||
stream->write(ptr, count);
|
||||
return count;
|
||||
}
|
||||
|
||||
curl_off_t Util::CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp)
|
||||
{
|
||||
curl_off_t realsize = size * nmemb;
|
||||
ChunkMemoryStruct *mem = (ChunkMemoryStruct *)userp;
|
||||
|
||||
mem->memory = (char *) realloc(mem->memory, mem->size + realsize + 1);
|
||||
if(mem->memory == NULL)
|
||||
{
|
||||
std::cout << "Not enough memory (realloc returned NULL)" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
memcpy(&(mem->memory[mem->size]), contents, realsize);
|
||||
mem->size += realsize;
|
||||
mem->memory[mem->size] = 0;
|
||||
|
||||
return realsize;
|
||||
}
|
||||
|
||||
@ -19,24 +19,7 @@ Website::Website()
|
||||
this->retries = 0;
|
||||
|
||||
curlhandle = curl_easy_init();
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, Globals::globalConfig.curlConf.sUserAgent.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, Globals::globalConfig.curlConf.iTimeout);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, Globals::globalConfig.curlConf.sCookiePath.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, Globals::globalConfig.curlConf.sCookiePath.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, Globals::globalConfig.curlConf.bVerifyPeer);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, Globals::globalConfig.curlConf.bVerbose);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, Globals::globalConfig.curlConf.iDownloadRate);
|
||||
|
||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, 30);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, 200);
|
||||
|
||||
if (!Globals::globalConfig.curlConf.sCACertPath.empty())
|
||||
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, Globals::globalConfig.curlConf.sCACertPath.c_str());
|
||||
Util::CurlHandleSetDefaultOptions(curlhandle, Globals::globalConfig.curlConf);
|
||||
}
|
||||
|
||||
Website::~Website()
|
||||
@ -44,34 +27,14 @@ Website::~Website()
|
||||
curl_easy_cleanup(curlhandle);
|
||||
}
|
||||
|
||||
size_t Website::writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp)
|
||||
{
|
||||
std::ostringstream *stream = (std::ostringstream*)userp;
|
||||
size_t count = size * nmemb;
|
||||
stream->write(ptr, count);
|
||||
return count;
|
||||
}
|
||||
|
||||
std::string Website::getResponse(const std::string& url)
|
||||
{
|
||||
std::ostringstream memory;
|
||||
std::string response;
|
||||
|
||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||
int max_retries = std::min(3, Globals::globalConfig.iRetries);
|
||||
|
||||
CURLcode result;
|
||||
do
|
||||
{
|
||||
if (Globals::globalConfig.iWait > 0)
|
||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
||||
result = curl_easy_perform(curlhandle);
|
||||
response = memory.str();
|
||||
memory.str(std::string());
|
||||
}
|
||||
while ((result != CURLE_OK) && response.empty() && (this->retries++ < Globals::globalConfig.iRetries));
|
||||
this->retries = 0; // reset retries counter
|
||||
CURLcode result = Util::CurlHandleGetResponse(curlhandle, response, max_retries);
|
||||
|
||||
if (result != CURLE_OK)
|
||||
{
|
||||
@ -373,7 +336,7 @@ int Website::Login(const std::string& email, const std::string& password)
|
||||
curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login_check");
|
||||
curl_easy_setopt(curlhandle, CURLOPT_POST, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0);
|
||||
@ -443,7 +406,7 @@ int Website::Login(const std::string& email, const std::string& password)
|
||||
curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login/two_step");
|
||||
curl_easy_setopt(curlhandle, CURLOPT_POST, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0);
|
||||
@ -597,7 +560,7 @@ bool Website::IsloggedInSimple()
|
||||
|
||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 0);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||
curl_easy_perform(curlhandle);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user