mirror of
https://github.com/Sude-/lgogdownloader.git
synced 2025-02-02 05:52:31 +01:00
Add retry support to --galaxy-install
Added retry support to --galaxy-install Changed some retry conditions Show reason for retry attempts Added some Curl related helper functions to Util
This commit is contained in:
parent
17bebfc374
commit
26315b23b8
@ -27,6 +27,7 @@
|
|||||||
#include "threadsafequeue.h"
|
#include "threadsafequeue.h"
|
||||||
#include "galaxyapi.h"
|
#include "galaxyapi.h"
|
||||||
#include "globals.h"
|
#include "globals.h"
|
||||||
|
#include "util.h"
|
||||||
|
|
||||||
#include <curl/curl.h>
|
#include <curl/curl.h>
|
||||||
#include <json/json.h>
|
#include <json/json.h>
|
||||||
@ -60,12 +61,6 @@ struct xferInfo
|
|||||||
curl_off_t offset;
|
curl_off_t offset;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ChunkMemoryStruct
|
|
||||||
{
|
|
||||||
char *memory;
|
|
||||||
curl_off_t size;
|
|
||||||
};
|
|
||||||
|
|
||||||
typedef struct
|
typedef struct
|
||||||
{
|
{
|
||||||
std::string filepath;
|
std::string filepath;
|
||||||
@ -142,7 +137,6 @@ class Downloader
|
|||||||
static void getGameDetailsThread(Config config, const unsigned int& tid);
|
static void getGameDetailsThread(Config config, const unsigned int& tid);
|
||||||
|
|
||||||
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
|
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
|
||||||
static size_t writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp);
|
|
||||||
static size_t writeData(void *ptr, size_t size, size_t nmemb, FILE *stream);
|
static size_t writeData(void *ptr, size_t size, size_t nmemb, FILE *stream);
|
||||||
static size_t readData(void *ptr, size_t size, size_t nmemb, FILE *stream);
|
static size_t readData(void *ptr, size_t size, size_t nmemb, FILE *stream);
|
||||||
|
|
||||||
|
@ -23,6 +23,13 @@
|
|||||||
#include <boost/regex.hpp>
|
#include <boost/regex.hpp>
|
||||||
#include <json/json.h>
|
#include <json/json.h>
|
||||||
#include <boost/date_time/posix_time/posix_time.hpp>
|
#include <boost/date_time/posix_time/posix_time.hpp>
|
||||||
|
#include <curl/curl.h>
|
||||||
|
|
||||||
|
typedef struct
|
||||||
|
{
|
||||||
|
char *memory;
|
||||||
|
curl_off_t size;
|
||||||
|
} ChunkMemoryStruct;
|
||||||
|
|
||||||
struct gameItem
|
struct gameItem
|
||||||
{
|
{
|
||||||
@ -77,6 +84,11 @@ namespace Util
|
|||||||
std::string getStrippedString(std::string str);
|
std::string getStrippedString(std::string str);
|
||||||
std::string makeEtaString(const unsigned long long& iBytesRemaining, const double& dlRate);
|
std::string makeEtaString(const unsigned long long& iBytesRemaining, const double& dlRate);
|
||||||
std::string makeEtaString(const boost::posix_time::time_duration& duration);
|
std::string makeEtaString(const boost::posix_time::time_duration& duration);
|
||||||
|
void CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf);
|
||||||
|
CURLcode CurlGetResponse(const std::string& url, std::string& response, int max_retries = -1);
|
||||||
|
CURLcode CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries = -1);
|
||||||
|
curl_off_t CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp);
|
||||||
|
curl_off_t CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp);
|
||||||
|
|
||||||
template<typename ... Args> std::string formattedString(const std::string& format, Args ... args)
|
template<typename ... Args> std::string formattedString(const std::string& format, Args ... args)
|
||||||
{
|
{
|
||||||
|
@ -27,7 +27,6 @@ class Website
|
|||||||
virtual ~Website();
|
virtual ~Website();
|
||||||
protected:
|
protected:
|
||||||
private:
|
private:
|
||||||
static size_t writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp);
|
|
||||||
CURL* curlhandle;
|
CURL* curlhandle;
|
||||||
bool IsloggedInSimple();
|
bool IsloggedInSimple();
|
||||||
bool IsLoggedInComplex(const std::string& email);
|
bool IsLoggedInComplex(const std::string& email);
|
||||||
|
@ -50,25 +50,6 @@ ThreadSafeQueue<zipFileEntry> dlQueueGalaxy_MojoSetupHack;
|
|||||||
std::mutex mtx_create_directories; // Mutex for creating directories in Downloader::processDownloadQueue
|
std::mutex mtx_create_directories; // Mutex for creating directories in Downloader::processDownloadQueue
|
||||||
std::atomic<unsigned long long> iTotalRemainingBytes(0);
|
std::atomic<unsigned long long> iTotalRemainingBytes(0);
|
||||||
|
|
||||||
static curl_off_t WriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp)
|
|
||||||
{
|
|
||||||
curl_off_t realsize = size * nmemb;
|
|
||||||
struct ChunkMemoryStruct *mem = (struct ChunkMemoryStruct *)userp;
|
|
||||||
|
|
||||||
mem->memory = (char *) realloc(mem->memory, mem->size + realsize + 1);
|
|
||||||
if(mem->memory == NULL)
|
|
||||||
{
|
|
||||||
std::cout << "Not enough memory (realloc returned NULL)" << std::endl;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
memcpy(&(mem->memory[mem->size]), contents, realsize);
|
|
||||||
mem->size += realsize;
|
|
||||||
mem->memory[mem->size] = 0;
|
|
||||||
|
|
||||||
return realsize;
|
|
||||||
}
|
|
||||||
|
|
||||||
Downloader::Downloader()
|
Downloader::Downloader()
|
||||||
{
|
{
|
||||||
if (Globals::globalConfig.bLogin)
|
if (Globals::globalConfig.bLogin)
|
||||||
@ -86,26 +67,13 @@ Downloader::Downloader()
|
|||||||
|
|
||||||
// Initialize curl and set curl options
|
// Initialize curl and set curl options
|
||||||
curlhandle = curl_easy_init();
|
curlhandle = curl_easy_init();
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
Util::CurlHandleSetDefaultOptions(curlhandle, Globals::globalConfig.curlConf);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, Globals::globalConfig.curlConf.sUserAgent.c_str());
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, Globals::globalConfig.curlConf.iTimeout);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, Globals::globalConfig.curlConf.bVerifyPeer);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, Globals::globalConfig.curlConf.bVerbose);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, Globals::globalConfig.curlConf.iDownloadRate);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_XFERINFODATA, this);
|
curl_easy_setopt(curlhandle, CURLOPT_XFERINFODATA, this);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||||
|
|
||||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, Globals::globalConfig.curlConf.iLowSpeedTimeout);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, Globals::globalConfig.curlConf.iLowSpeedTimeoutRate);
|
|
||||||
|
|
||||||
if (!Globals::globalConfig.curlConf.sCACertPath.empty())
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, Globals::globalConfig.curlConf.sCACertPath.c_str());
|
|
||||||
|
|
||||||
// Create new GOG website handle
|
// Create new GOG website handle
|
||||||
gogWebsite = new Website();
|
gogWebsite = new Website();
|
||||||
@ -1513,21 +1481,9 @@ std::string Downloader::getResponse(const std::string& url)
|
|||||||
std::string response;
|
std::string response;
|
||||||
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeMemoryCallback);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
|
||||||
|
|
||||||
CURLcode result;
|
int max_retries = std::min(3, Globals::globalConfig.iRetries);
|
||||||
do
|
CURLcode result = Util::CurlHandleGetResponse(curlhandle, response, max_retries);
|
||||||
{
|
|
||||||
if (Globals::globalConfig.iWait > 0)
|
|
||||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
|
||||||
result = curl_easy_perform(curlhandle);
|
|
||||||
response = memory.str();
|
|
||||||
memory.str(std::string());
|
|
||||||
}
|
|
||||||
while ((result != CURLE_OK) && response.empty() && (this->retries++ < Globals::globalConfig.iRetries));
|
|
||||||
this->retries = 0; // reset retries counter
|
|
||||||
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0);
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0);
|
||||||
@ -1645,13 +1601,6 @@ int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t d
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t Downloader::writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp) {
|
|
||||||
std::ostringstream *stream = (std::ostringstream*)userp;
|
|
||||||
size_t count = size * nmemb;
|
|
||||||
stream->write(ptr, count);
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t Downloader::writeData(void *ptr, size_t size, size_t nmemb, FILE *stream)
|
size_t Downloader::writeData(void *ptr, size_t size, size_t nmemb, FILE *stream)
|
||||||
{
|
{
|
||||||
return fwrite(ptr, size, nmemb, stream);
|
return fwrite(ptr, size, nmemb, stream);
|
||||||
@ -2552,27 +2501,12 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
|||||||
}
|
}
|
||||||
|
|
||||||
CURL* dlhandle = curl_easy_init();
|
CURL* dlhandle = curl_easy_init();
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.curlConf.sUserAgent.c_str());
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOSIGNAL, 1);
|
|
||||||
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.curlConf.iTimeout);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_SSL_VERIFYPEER, conf.curlConf.bVerifyPeer);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_VERBOSE, conf.curlConf.bVerbose);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.curlConf.iDownloadRate);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||||
|
|
||||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_TIME, conf.curlConf.iLowSpeedTimeout);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.curlConf.iLowSpeedTimeoutRate);
|
|
||||||
|
|
||||||
if (!conf.curlConf.sCACertPath.empty())
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_CAINFO, conf.curlConf.sCACertPath.c_str());
|
|
||||||
|
|
||||||
xferInfo xferinfo;
|
xferInfo xferinfo;
|
||||||
xferinfo.tid = tid;
|
xferinfo.tid = tid;
|
||||||
xferinfo.curlhandle = dlhandle;
|
xferinfo.curlhandle = dlhandle;
|
||||||
@ -2804,6 +2738,7 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
|||||||
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
|
||||||
long int response_code = 0;
|
long int response_code = 0;
|
||||||
bool bShouldRetry = false;
|
bool bShouldRetry = false;
|
||||||
|
std::string retry_reason;
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
if (conf.iWait > 0)
|
if (conf.iWait > 0)
|
||||||
@ -2812,7 +2747,13 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
|||||||
response_code = 0; // Make sure that response code is reset
|
response_code = 0; // Make sure that response code is reset
|
||||||
|
|
||||||
if (iRetryCount != 0)
|
if (iRetryCount != 0)
|
||||||
msgQueue.push(Message("Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix));
|
{
|
||||||
|
std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string();
|
||||||
|
if (!retry_reason.empty())
|
||||||
|
retry_msg += " (" + retry_reason + ")";
|
||||||
|
msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix));
|
||||||
|
}
|
||||||
|
retry_reason = ""; // reset retry reason
|
||||||
|
|
||||||
FILE* outfile;
|
FILE* outfile;
|
||||||
// File exists, resume
|
// File exists, resume
|
||||||
@ -2875,6 +2816,7 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid)
|
|||||||
if (bShouldRetry)
|
if (bShouldRetry)
|
||||||
{
|
{
|
||||||
iRetryCount++;
|
iRetryCount++;
|
||||||
|
retry_reason = std::string(curl_easy_strerror(result));
|
||||||
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
|
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
|
||||||
bResume = true;
|
bResume = true;
|
||||||
}
|
}
|
||||||
@ -3649,27 +3591,12 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
|||||||
}
|
}
|
||||||
|
|
||||||
CURL* dlhandle = curl_easy_init();
|
CURL* dlhandle = curl_easy_init();
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
Util::CurlHandleSetDefaultOptions(dlhandle, Globals::globalConfig.curlConf);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.curlConf.sUserAgent.c_str());
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOSIGNAL, 1);
|
|
||||||
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.curlConf.iTimeout);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_SSL_VERIFYPEER, conf.curlConf.bVerifyPeer);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_VERBOSE, conf.curlConf.bVerbose);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.curlConf.iDownloadRate);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||||
|
|
||||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_TIME, conf.curlConf.iLowSpeedTimeout);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.curlConf.iLowSpeedTimeoutRate);
|
|
||||||
|
|
||||||
if (!conf.curlConf.sCACertPath.empty())
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_CAINFO, conf.curlConf.sCACertPath.c_str());
|
|
||||||
|
|
||||||
xferInfo xferinfo;
|
xferInfo xferinfo;
|
||||||
xferinfo.tid = tid;
|
xferinfo.tid = tid;
|
||||||
xferinfo.curlhandle = dlhandle;
|
xferinfo.curlhandle = dlhandle;
|
||||||
@ -3829,10 +3756,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
|||||||
std::time_t timestamp = -1;
|
std::time_t timestamp = -1;
|
||||||
for (unsigned int j = start_chunk; j < item.chunks.size(); ++j)
|
for (unsigned int j = start_chunk; j < item.chunks.size(); ++j)
|
||||||
{
|
{
|
||||||
ChunkMemoryStruct chunk;
|
|
||||||
chunk.memory = (char *) malloc(1);
|
|
||||||
chunk.size = 0;
|
|
||||||
|
|
||||||
// Refresh Galaxy login if token is expired
|
// Refresh Galaxy login if token is expired
|
||||||
if (galaxy->isTokenExpired())
|
if (galaxy->isTokenExpired())
|
||||||
{
|
{
|
||||||
@ -3840,7 +3763,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
|||||||
{
|
{
|
||||||
msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix));
|
msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix));
|
||||||
vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
|
vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
|
||||||
free(chunk.memory);
|
|
||||||
delete galaxy;
|
delete galaxy;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -3857,7 +3779,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
|||||||
bChunkFailure = true;
|
bChunkFailure = true;
|
||||||
std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")";
|
std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")";
|
||||||
msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix));
|
msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix));
|
||||||
free(chunk.memory);
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3925,7 +3846,6 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
|||||||
{
|
{
|
||||||
bChunkFailure = true;
|
bChunkFailure = true;
|
||||||
msgQueue.push(Message(path.string() + ": Failed to get download url", MSGTYPE_ERROR, msg_prefix));
|
msgQueue.push(Message(path.string() + ": Failed to get download url", MSGTYPE_ERROR, msg_prefix));
|
||||||
free(chunk.memory);
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3940,25 +3860,78 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
|
|||||||
// Select url with lowest priority score
|
// Select url with lowest priority score
|
||||||
std::string url = cdnUrls[0].url;
|
std::string url = cdnUrls[0].url;
|
||||||
|
|
||||||
|
ChunkMemoryStruct chunk;
|
||||||
|
chunk.memory = (char *) malloc(1);
|
||||||
|
chunk.size = 0;
|
||||||
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, WriteChunkMemoryCallback);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteChunkMemoryCallback);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &chunk);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &chunk);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread);
|
curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo);
|
curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||||
|
curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, 0);
|
||||||
|
|
||||||
std::string filepath_and_chunk = path.string() + " (chunk " + std::to_string(j + 1) + "/" + std::to_string(item.chunks.size()) + ")";
|
std::string filepath_and_chunk = path.string() + " (chunk " + std::to_string(j + 1) + "/" + std::to_string(item.chunks.size()) + ")";
|
||||||
vDownloadInfo[tid].setFilename(filepath_and_chunk);
|
vDownloadInfo[tid].setFilename(filepath_and_chunk);
|
||||||
|
|
||||||
|
CURLcode result;
|
||||||
|
int iRetryCount = 0;
|
||||||
|
long int response_code = 0;
|
||||||
|
bool bShouldRetry = false;
|
||||||
|
std::string retry_reason;
|
||||||
|
do
|
||||||
|
{
|
||||||
if (Globals::globalConfig.iWait > 0)
|
if (Globals::globalConfig.iWait > 0)
|
||||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
||||||
|
|
||||||
xferinfo.offset = 0;
|
response_code = 0; // Make sure that response code is reset
|
||||||
|
|
||||||
|
if (iRetryCount != 0)
|
||||||
|
{
|
||||||
|
std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath_and_chunk;
|
||||||
|
if (!retry_reason.empty())
|
||||||
|
retry_msg += " (" + retry_reason + ")";
|
||||||
|
msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix));
|
||||||
|
|
||||||
|
curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, chunk.size);
|
||||||
|
}
|
||||||
|
retry_reason = ""; // reset retry reason
|
||||||
|
|
||||||
|
xferinfo.offset = chunk.size;
|
||||||
xferinfo.timer.reset();
|
xferinfo.timer.reset();
|
||||||
xferinfo.TimeAndSize.clear();
|
xferinfo.TimeAndSize.clear();
|
||||||
|
result = curl_easy_perform(dlhandle);
|
||||||
|
|
||||||
CURLcode result = curl_easy_perform(dlhandle);
|
switch (result)
|
||||||
|
{
|
||||||
|
// Retry on these errors
|
||||||
|
case CURLE_PARTIAL_FILE:
|
||||||
|
case CURLE_OPERATION_TIMEDOUT:
|
||||||
|
case CURLE_RECV_ERROR:
|
||||||
|
bShouldRetry = true;
|
||||||
|
break;
|
||||||
|
// Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable"
|
||||||
|
case CURLE_HTTP_RETURNED_ERROR:
|
||||||
|
curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code);
|
||||||
|
if (response_code == 416)
|
||||||
|
bShouldRetry = false;
|
||||||
|
else
|
||||||
|
bShouldRetry = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
bShouldRetry = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bShouldRetry)
|
||||||
|
{
|
||||||
|
iRetryCount++;
|
||||||
|
retry_reason = std::string(curl_easy_strerror(result));
|
||||||
|
}
|
||||||
|
|
||||||
|
} while (bShouldRetry && (iRetryCount <= conf.iRetries));
|
||||||
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||||
@ -4277,7 +4250,7 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
|
|||||||
std::stringstream splitfiles_uncompressed;
|
std::stringstream splitfiles_uncompressed;
|
||||||
|
|
||||||
CURLcode result = CURLE_RECV_ERROR;
|
CURLcode result = CURLE_RECV_ERROR;
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &splitfiles_compressed);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &splitfiles_compressed);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, dlrange.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_RANGE, dlrange.c_str());
|
||||||
result = curl_easy_perform(curlhandle);
|
result = curl_easy_perform(curlhandle);
|
||||||
@ -4581,27 +4554,12 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
|
|||||||
std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
|
std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
|
||||||
|
|
||||||
CURL* dlhandle = curl_easy_init();
|
CURL* dlhandle = curl_easy_init();
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.curlConf.sUserAgent.c_str());
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_NOSIGNAL, 1);
|
|
||||||
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.curlConf.iTimeout);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_SSL_VERIFYPEER, conf.curlConf.bVerifyPeer);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_VERBOSE, conf.curlConf.bVerbose);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.curlConf.iDownloadRate);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
|
||||||
|
|
||||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_TIME, conf.curlConf.iLowSpeedTimeout);
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.curlConf.iLowSpeedTimeoutRate);
|
|
||||||
|
|
||||||
if (!conf.curlConf.sCACertPath.empty())
|
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_CAINFO, conf.curlConf.sCACertPath.c_str());
|
|
||||||
|
|
||||||
xferInfo xferinfo;
|
xferInfo xferinfo;
|
||||||
xferinfo.tid = tid;
|
xferinfo.tid = tid;
|
||||||
xferinfo.curlhandle = dlhandle;
|
xferinfo.curlhandle = dlhandle;
|
||||||
@ -4743,7 +4701,7 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
|
|||||||
std::string link_target;
|
std::string link_target;
|
||||||
|
|
||||||
CURLcode result = CURLE_RECV_ERROR;
|
CURLcode result = CURLE_RECV_ERROR;
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &symlink_compressed);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &symlink_compressed);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str());
|
curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str());
|
||||||
|
|
||||||
@ -4826,7 +4784,7 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
|
|||||||
|
|
||||||
std::stringstream data_compressed;
|
std::stringstream data_compressed;
|
||||||
vDownloadInfo[tid].setFilename(path.string());
|
vDownloadInfo[tid].setFilename(path.string());
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &data_compressed);
|
curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &data_compressed);
|
||||||
curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str());
|
curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str());
|
||||||
|
|
||||||
@ -5111,7 +5069,7 @@ int Downloader::mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEn
|
|||||||
std::stringstream head;
|
std::stringstream head;
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_URL, installer_url.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_URL, installer_url.c_str());
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &head);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &head);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, head_range.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_RANGE, head_range.c_str());
|
||||||
result = curl_easy_perform(curlhandle);
|
result = curl_easy_perform(curlhandle);
|
||||||
@ -5142,7 +5100,7 @@ int Downloader::mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEn
|
|||||||
// Get tail
|
// Get tail
|
||||||
std::stringstream tail;
|
std::stringstream tail;
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str());
|
||||||
result = curl_easy_perform(curlhandle);
|
result = curl_easy_perform(curlhandle);
|
||||||
@ -5186,7 +5144,7 @@ int Downloader::mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEn
|
|||||||
tail.str(std::string());
|
tail.str(std::string());
|
||||||
tail_range = std::to_string(mojosetup_cd_offset) + "-" + std::to_string(file_size);
|
tail_range = std::to_string(mojosetup_cd_offset) + "-" + std::to_string(file_size);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str());
|
||||||
result = curl_easy_perform(curlhandle);
|
result = curl_easy_perform(curlhandle);
|
||||||
|
@ -26,25 +26,7 @@ galaxyAPI::galaxyAPI(CurlConfig& conf)
|
|||||||
this->curlConf = conf;
|
this->curlConf = conf;
|
||||||
|
|
||||||
curlhandle = curl_easy_init();
|
curlhandle = curl_easy_init();
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
Util::CurlHandleSetDefaultOptions(curlhandle, this->curlConf);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_PROGRESSDATA, this);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, curlConf.iTimeout);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, curlConf.sCookiePath.c_str());
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, curlConf.sCookiePath.c_str());
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, curlConf.bVerifyPeer);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, curlConf.bVerbose);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, curlConf.iDownloadRate);
|
|
||||||
|
|
||||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, curlConf.iLowSpeedTimeout);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, curlConf.iLowSpeedTimeoutRate);
|
|
||||||
|
|
||||||
if (!curlConf.sCACertPath.empty())
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, curlConf.sCACertPath.c_str());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
galaxyAPI::~galaxyAPI()
|
galaxyAPI::~galaxyAPI()
|
||||||
@ -99,8 +81,6 @@ bool galaxyAPI::isTokenExpired()
|
|||||||
|
|
||||||
std::string galaxyAPI::getResponse(const std::string& url, const bool& zlib_decompress)
|
std::string galaxyAPI::getResponse(const std::string& url, const bool& zlib_decompress)
|
||||||
{
|
{
|
||||||
std::ostringstream memory;
|
|
||||||
|
|
||||||
struct curl_slist *header = NULL;
|
struct curl_slist *header = NULL;
|
||||||
|
|
||||||
std::string access_token;
|
std::string access_token;
|
||||||
@ -112,24 +92,11 @@ std::string galaxyAPI::getResponse(const std::string& url, const bool& zlib_deco
|
|||||||
header = curl_slist_append(header, bearer.c_str());
|
header = curl_slist_append(header, bearer.c_str());
|
||||||
}
|
}
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, header);
|
curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, header);
|
||||||
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, galaxyAPI::writeMemoryCallback);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
|
||||||
|
|
||||||
int retries = 0;
|
int max_retries = std::min(3, Globals::globalConfig.iRetries);
|
||||||
CURLcode result;
|
|
||||||
std::string response;
|
std::string response;
|
||||||
do
|
Util::CurlHandleGetResponse(curlhandle, response, max_retries);
|
||||||
{
|
|
||||||
if (Globals::globalConfig.iWait > 0)
|
|
||||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
|
||||||
result = curl_easy_perform(curlhandle);
|
|
||||||
response = memory.str();
|
|
||||||
memory.str(std::string());
|
|
||||||
}
|
|
||||||
while ((result != CURLE_OK) && response.empty() && (retries++ < Globals::globalConfig.iRetries));
|
|
||||||
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, NULL);
|
curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, NULL);
|
||||||
curl_slist_free_all(header);
|
curl_slist_free_all(header);
|
||||||
|
116
src/util.cpp
116
src/util.cpp
@ -732,3 +732,119 @@ std::string Util::makeEtaString(const boost::posix_time::time_duration& duration
|
|||||||
|
|
||||||
return etastr;
|
return etastr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void Util::CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf)
|
||||||
|
{
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, conf.sUserAgent.c_str());
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, conf.iTimeout);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, conf.sCookiePath.c_str());
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, conf.sCookiePath.c_str());
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, conf.bVerifyPeer);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, conf.bVerbose);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.iDownloadRate);
|
||||||
|
|
||||||
|
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, conf.iLowSpeedTimeout);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.iLowSpeedTimeoutRate);
|
||||||
|
|
||||||
|
if (!conf.sCACertPath.empty())
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, conf.sCACertPath.c_str());
|
||||||
|
}
|
||||||
|
|
||||||
|
CURLcode Util::CurlGetResponse(const std::string& url, std::string& response, int max_retries)
|
||||||
|
{
|
||||||
|
CURLcode result;
|
||||||
|
CURL *handle = curl_easy_init();
|
||||||
|
Util::CurlHandleSetDefaultOptions(handle, Globals::globalConfig.curlConf);
|
||||||
|
curl_easy_setopt(handle, CURLOPT_URL, url.c_str());
|
||||||
|
|
||||||
|
result = Util::CurlHandleGetResponse(handle, response, max_retries);
|
||||||
|
|
||||||
|
curl_easy_cleanup(handle);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
CURLcode Util::CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries)
|
||||||
|
{
|
||||||
|
CURLcode result;
|
||||||
|
int retries = 0;
|
||||||
|
std::ostringstream memory;
|
||||||
|
bool bShouldRetry = false;
|
||||||
|
long int response_code = 0;
|
||||||
|
if (max_retries < 0)
|
||||||
|
max_retries = Globals::globalConfig.iRetries;
|
||||||
|
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||||
|
|
||||||
|
do
|
||||||
|
{
|
||||||
|
if (bShouldRetry)
|
||||||
|
retries++;
|
||||||
|
|
||||||
|
if (Globals::globalConfig.iWait > 0)
|
||||||
|
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
||||||
|
result = curl_easy_perform(curlhandle);
|
||||||
|
response = memory.str();
|
||||||
|
memory.str(std::string());
|
||||||
|
|
||||||
|
switch (result)
|
||||||
|
{
|
||||||
|
// Retry on these errors
|
||||||
|
case CURLE_PARTIAL_FILE:
|
||||||
|
case CURLE_OPERATION_TIMEDOUT:
|
||||||
|
case CURLE_RECV_ERROR:
|
||||||
|
bShouldRetry = true;
|
||||||
|
break;
|
||||||
|
// Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "404 Not Found"
|
||||||
|
case CURLE_HTTP_RETURNED_ERROR:
|
||||||
|
curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code);
|
||||||
|
if (response_code == 404)
|
||||||
|
bShouldRetry = false;
|
||||||
|
else
|
||||||
|
bShouldRetry = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
bShouldRetry = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (retries >= max_retries)
|
||||||
|
bShouldRetry = false;
|
||||||
|
} while (bShouldRetry);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
curl_off_t Util::CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp)
|
||||||
|
{
|
||||||
|
std::ostringstream *stream = (std::ostringstream*)userp;
|
||||||
|
std::streamsize count = (std::streamsize) size * nmemb;
|
||||||
|
stream->write(ptr, count);
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
curl_off_t Util::CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp)
|
||||||
|
{
|
||||||
|
curl_off_t realsize = size * nmemb;
|
||||||
|
ChunkMemoryStruct *mem = (ChunkMemoryStruct *)userp;
|
||||||
|
|
||||||
|
mem->memory = (char *) realloc(mem->memory, mem->size + realsize + 1);
|
||||||
|
if(mem->memory == NULL)
|
||||||
|
{
|
||||||
|
std::cout << "Not enough memory (realloc returned NULL)" << std::endl;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
memcpy(&(mem->memory[mem->size]), contents, realsize);
|
||||||
|
mem->size += realsize;
|
||||||
|
mem->memory[mem->size] = 0;
|
||||||
|
|
||||||
|
return realsize;
|
||||||
|
}
|
||||||
|
@ -19,24 +19,7 @@ Website::Website()
|
|||||||
this->retries = 0;
|
this->retries = 0;
|
||||||
|
|
||||||
curlhandle = curl_easy_init();
|
curlhandle = curl_easy_init();
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
|
Util::CurlHandleSetDefaultOptions(curlhandle, Globals::globalConfig.curlConf);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, Globals::globalConfig.curlConf.sUserAgent.c_str());
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, Globals::globalConfig.curlConf.iTimeout);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, Globals::globalConfig.curlConf.sCookiePath.c_str());
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, Globals::globalConfig.curlConf.sCookiePath.c_str());
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, Globals::globalConfig.curlConf.bVerifyPeer);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, Globals::globalConfig.curlConf.bVerbose);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, Globals::globalConfig.curlConf.iDownloadRate);
|
|
||||||
|
|
||||||
// Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, 30);
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, 200);
|
|
||||||
|
|
||||||
if (!Globals::globalConfig.curlConf.sCACertPath.empty())
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_CAINFO, Globals::globalConfig.curlConf.sCACertPath.c_str());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Website::~Website()
|
Website::~Website()
|
||||||
@ -44,34 +27,14 @@ Website::~Website()
|
|||||||
curl_easy_cleanup(curlhandle);
|
curl_easy_cleanup(curlhandle);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t Website::writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp)
|
|
||||||
{
|
|
||||||
std::ostringstream *stream = (std::ostringstream*)userp;
|
|
||||||
size_t count = size * nmemb;
|
|
||||||
stream->write(ptr, count);
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string Website::getResponse(const std::string& url)
|
std::string Website::getResponse(const std::string& url)
|
||||||
{
|
{
|
||||||
std::ostringstream memory;
|
|
||||||
std::string response;
|
std::string response;
|
||||||
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
int max_retries = std::min(3, Globals::globalConfig.iRetries);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
|
||||||
|
|
||||||
CURLcode result;
|
CURLcode result = Util::CurlHandleGetResponse(curlhandle, response, max_retries);
|
||||||
do
|
|
||||||
{
|
|
||||||
if (Globals::globalConfig.iWait > 0)
|
|
||||||
usleep(Globals::globalConfig.iWait); // Delay the request by specified time
|
|
||||||
result = curl_easy_perform(curlhandle);
|
|
||||||
response = memory.str();
|
|
||||||
memory.str(std::string());
|
|
||||||
}
|
|
||||||
while ((result != CURLE_OK) && response.empty() && (this->retries++ < Globals::globalConfig.iRetries));
|
|
||||||
this->retries = 0; // reset retries counter
|
|
||||||
|
|
||||||
if (result != CURLE_OK)
|
if (result != CURLE_OK)
|
||||||
{
|
{
|
||||||
@ -373,7 +336,7 @@ int Website::Login(const std::string& email, const std::string& password)
|
|||||||
curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login_check");
|
curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login_check");
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_POST, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_POST, 1);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str());
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0);
|
curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0);
|
||||||
@ -443,7 +406,7 @@ int Website::Login(const std::string& email, const std::string& password)
|
|||||||
curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login/two_step");
|
curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login/two_step");
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_POST, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_POST, 1);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str());
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0);
|
curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0);
|
||||||
@ -597,7 +560,7 @@ bool Website::IsloggedInSimple()
|
|||||||
|
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 0);
|
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 0);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Website::writeMemoryCallback);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
|
||||||
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
|
||||||
curl_easy_perform(curlhandle);
|
curl_easy_perform(curlhandle);
|
||||||
|
Loading…
x
Reference in New Issue
Block a user