Mirrors/lgogdownloader
2
0
Fork 0
mirror of https://github.com/Sude-/lgogdownloader.git synced 2024-11-20 03:39:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

Upload local saves to the cloud

Browse Source
This commit is contained in:
loki-47-6F-64 2022-08-04 16:57:15 +02:00
parent 829d263bc7
commit 33ec0434aa
5 changed files with 304 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/downloader.h
View File

@ -102,6 +102,9 @@ class Downloader
void repair();
void download();
void downloadCloudSaves(const std::string& product_id, int build_index = -1);
void downloadCloudSavesById(const std::string& product_id, int build_index = -1);
void uploadCloudSaves(const std::string& product_id, int build_index = -1);
void uploadCloudSavesById(const std::string& product_id, int build_index = -1);
void checkOrphans();
void checkStatus();
void updateCache();
@ -144,6 +147,7 @@ class Downloader
void saveChangelog(const std::string& changelog, const std::string& filepath);
static void processDownloadQueue(Config conf, const unsigned int& tid);
static void processCloudSaveDownloadQueue(Config conf, const unsigned int& tid);
static void processCloudSaveUploadQueue(Config conf, const unsigned int& tid);
static int progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow);
template <typename T> void printProgress(const ThreadSafeQueue<T>& download_queue);
static void getGameDetailsThread(Config config, const unsigned int& tid);

1
include/util.h
View File

@ -91,6 +91,7 @@ namespace Util
CURLcode CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries = -1);
curl_off_t CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp);
curl_off_t CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp);
curl_off_t CurlReadChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, ChunkMemoryStruct *userp);
template<typename ... Args> std::string formattedString(const std::string& format, Args ... args)
{

12
main.cpp
View File

@ -166,6 +166,7 @@ int main(int argc, char *argv[])
std::string galaxy_product_id_show_cloud_paths;
std::string galaxy_product_id_show_local_cloud_paths;
std::string galaxy_product_cloud_saves;
std::string galaxy_upload_product_cloud_saves;
std::string tags;
std::vector<std::string> vFileIdStrings;
@ -288,6 +289,7 @@ int main(int argc, char *argv[])
("galaxy-install", bpo::value<std::string>(&galaxy_product_id_install)->default_value(""), "Install game using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
("galaxy-show-builds", bpo::value<std::string>(&galaxy_product_id_show_builds)->default_value(""), "Show game builds using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
("galaxy-download-cloud-saves", bpo::value<std::string>(&galaxy_product_cloud_saves)->default_value(""), "Download cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
("galaxy-upload-cloud-saves", bpo::value<std::string>(&galaxy_upload_product_cloud_saves)->default_value(""), "Upload cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
("galaxy-show-cloud-saves", bpo::value<std::string>(&galaxy_product_id_show_cloud_paths)->default_value(""), "Show game cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
("galaxy-show-local-cloud-saves", bpo::value<std::string>(&galaxy_product_id_show_local_cloud_paths)->default_value(""), "Show local cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345")
("galaxy-platform", bpo::value<std::string>(&sGalaxyPlatform)->default_value("w"), galaxy_platform_text.c_str())
@ -832,6 +834,16 @@ int main(int argc, char *argv[])
}
downloader.downloadCloudSaves(product_id, build_index);
}
else if (!galaxy_upload_product_cloud_saves.empty()) {
int build_index = -1;
std::vector<std::string> tokens = Util::tokenize(galaxy_upload_product_cloud_saves, "/");
std::string product_id = tokens[0];
if (tokens.size() == 2)
{
build_index = std::stoi(tokens[1]);
}
downloader.uploadCloudSaves(product_id, build_index);
}
else
{
if (!Globals::globalConfig.bLogin)

295
src/downloader.cpp
View File

@ -64,6 +64,25 @@ std::string username() {
return user ? user : std::string();
}
void dirForEachHelper(const boost::filesystem::path &location, std::function<void(boost::filesystem::directory_iterator)> &f) {
boost::filesystem::directory_iterator begin { location };
boost::filesystem::directory_iterator end;
for(boost::filesystem::directory_iterator curr_dir { begin }; curr_dir != end; ++curr_dir) {
if(boost::filesystem::is_directory(*curr_dir)) {
dirForEachHelper(*curr_dir, f);
}
else {
f(curr_dir);
}
}
}
void dirForEach(const std::string &location, std::function<void(boost::filesystem::directory_iterator)> &&f) {
dirForEachHelper(location, f);
}
Downloader::Downloader()
{
if (Globals::globalConfig.bLogin)
@ -2530,6 +2549,149 @@ void Downloader::showWishlist()
return;
}
void Downloader::processCloudSaveUploadQueue(Config conf, const unsigned int& tid) {
std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
std::unique_ptr<galaxyAPI> galaxy { new galaxyAPI(Globals::globalConfig.curlConf) };
if (!galaxy->init())
{
if (!galaxy->refreshLogin())
{
msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix));
vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
return;
}
}
CURL* dlhandle = curl_easy_init();
Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf);
curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Util::CurlReadChunkMemoryCallback);
curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L);
xferInfo xferinfo;
xferinfo.tid = tid;
xferinfo.curlhandle = dlhandle;
curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread);
curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo);
cloudSaveFile csf;
std::string access_token;
if (!Globals::galaxyConf.isExpired()) {
access_token = Globals::galaxyConf.getAccessToken();
}
if (access_token.empty()) {
return;
}
std::string bearer = "Authorization: Bearer " + access_token;
while(dlCloudSaveQueue.try_pop(csf)) {
CURLcode result = CURLE_RECV_ERROR; // assume network error
int iRetryCount = 0;
iTotalRemainingBytes.fetch_sub(csf.fileSize);
vDownloadInfo[tid].setFilename(csf.location);
std::string filecontents;
{
std::ifstream in { csf.location, std::ios_base::in | std::ios_base::binary };
in >> filecontents;
}
ChunkMemoryStruct cms {
&filecontents[0],
(curl_off_t)filecontents.size()
};
auto md5 = Util::getChunkHash((std::uint8_t*)filecontents.data(), filecontents.size(), RHASH_MD5);
auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path;
curl_slist *header = nullptr;
header = curl_slist_append(header, bearer.c_str());
header = curl_slist_append(header, ("X-Object-Meta-LocalLastModified: " + boost::posix_time::to_iso_extended_string(csf.lastModified)).c_str());
header = curl_slist_append(header, ("Etag: " + md5).c_str());
header = curl_slist_append(header, "Content-Type: Octet-Stream");
header = curl_slist_append(header, ("Content-Length: " + std::to_string(filecontents.size())).c_str());
curl_easy_setopt(dlhandle, CURLOPT_PUT, 1L);
curl_easy_setopt(dlhandle, CURLOPT_CUSTOMREQUEST, "PUT");
curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header);
curl_easy_setopt(dlhandle, CURLOPT_READDATA, &cms);
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
msgQueue.push(Message("Begin upload: " + csf.location, MSGTYPE_INFO, msg_prefix));
bool bShouldRetry = false;
long int response_code = 0;
std::string retry_reason;
do
{
if (conf.iWait > 0)
usleep(conf.iWait); // Wait before continuing
response_code = 0; // Make sure that response code is reset
if (iRetryCount != 0)
{
std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + boost::filesystem::path(csf.location).filename().string();
if (!retry_reason.empty())
retry_msg += " (" + retry_reason + ")";
msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix));
}
retry_reason = ""; // reset retry reason
xferinfo.offset = 0;
xferinfo.timer.reset();
xferinfo.TimeAndSize.clear();
result = curl_easy_perform(dlhandle);
switch (result)
{
// Retry on these errors
case CURLE_PARTIAL_FILE:
case CURLE_OPERATION_TIMEDOUT:
case CURLE_RECV_ERROR:
case CURLE_SSL_CONNECT_ERROR:
bShouldRetry = true;
break;
// Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable"
case CURLE_HTTP_RETURNED_ERROR:
curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code);
if (response_code == 416 || response_code == 422 || response_code == 400 || response_code == 422) {
msgQueue.push(Message(std::to_string(response_code) + ": " + curl_easy_strerror(result)));
bShouldRetry = false;
}
else
bShouldRetry = true;
break;
default:
bShouldRetry = false;
break;
}
if (bShouldRetry) {
iRetryCount++;
retry_reason = std::to_string(response_code) + ": " + curl_easy_strerror(result);
}
} while (bShouldRetry && (iRetryCount <= conf.iRetries));
curl_slist_free_all(header);
}
curl_easy_cleanup(dlhandle);
vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED);
msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix));
}
void Downloader::processCloudSaveDownloadQueue(Config conf, const unsigned int& tid) {
std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
@ -2621,7 +2783,6 @@ void Downloader::processCloudSaveDownloadQueue(Config conf, const unsigned int&
}
auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path;
msgQueue.push(Message(url));
curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header);
curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str());
long int response_code = 0;
@ -4400,6 +4561,26 @@ std::pair<std::string::const_iterator, std::string::const_iterator> getline(std:
return { end, end };
}
void Downloader::uploadCloudSaves(const std::string& product_id, int build_index)
{
std::string id;
if(this->galaxySelectProductIdHelper(product_id, id))
{
if (!id.empty())
this->uploadCloudSavesById(id, build_index);
}
}
void Downloader::downloadCloudSaves(const std::string& product_id, int build_index)
{
std::string id;
if(this->galaxySelectProductIdHelper(product_id, id))
{
if (!id.empty())
this->downloadCloudSavesById(id, build_index);
}
}
void Downloader::galaxyShowCloudSaves(const std::string& product_id, int build_index)
{
std::string id;
@ -4533,7 +4714,96 @@ int Downloader::cloudSaveListByIdForEach(const std::string& product_id, int buil
return 0;
}
void Downloader::downloadCloudSaves(const std::string& product_id, int build_index)
void Downloader::uploadCloudSavesById(const std::string& product_id, int build_index)
{
auto name_to_locations = cloudSaveLocations(product_id, build_index);
if(name_to_locations.empty()) {
std::cout << "Cloud saves not supported for this game" << std::endl;
}
std::map<std::string, cloudSaveFile> path_to_cloudSaveFile;
for(auto &name_to_location : name_to_locations) {
auto &name = name_to_location.first;
auto &location = name_to_location.second;
if(!boost::filesystem::exists(location) || !boost::filesystem::is_directory(location)) {
continue;
}
dirForEach(location, [&](boost::filesystem::directory_iterator file) {
cloudSaveFile csf {
boost::posix_time::from_time_t(boost::filesystem::last_write_time(*file) - 1),
boost::filesystem::file_size(*file),
(name / boost::filesystem::relative(*file, location)).string(),
file->path().string()
};
path_to_cloudSaveFile.insert(std::make_pair(csf.path, std::move(csf)));
});
}
if(path_to_cloudSaveFile.empty()) {
std::cout << "No local cloud saves found" << std::endl;
return;
}
auto res = this->cloudSaveListByIdForEach(product_id, build_index, [&](cloudSaveFile &csf) {
auto it = path_to_cloudSaveFile.find(csf.path);
//If remote save is not locally stored, skip
if(it == std::end(path_to_cloudSaveFile)) {
return;
}
cloudSaveFile local_csf { std::move(it->second) };
path_to_cloudSaveFile.erase(it);
if(csf.lastModified < local_csf.lastModified || boost::filesystem::path(csf.location).filename().string() == "test.txt") {
iTotalRemainingBytes.fetch_add(local_csf.fileSize);
dlCloudSaveQueue.push(local_csf);
}
});
for(auto &path_csf : path_to_cloudSaveFile) {
auto &csf = path_csf.second;
iTotalRemainingBytes.fetch_add(csf.fileSize);
dlCloudSaveQueue.push(csf);
}
if(res || dlCloudSaveQueue.empty()) {
return;
}
// Limit thread count to number of items in upload queue
unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast<unsigned int>(dlCloudSaveQueue.size()));
// Create download threads
std::vector<std::thread> vThreads;
for (unsigned int i = 0; i < iThreads; ++i)
{
DownloadInfo dlInfo;
dlInfo.setStatus(DLSTATUS_NOTSTARTED);
vDownloadInfo.push_back(dlInfo);
vThreads.push_back(std::thread(Downloader::processCloudSaveUploadQueue, Globals::globalConfig, i));
}
this->printProgress(dlCloudSaveQueue);
// Join threads
for (unsigned int i = 0; i < vThreads.size(); ++i) {
vThreads[i].join();
}
vThreads.clear();
vDownloadInfo.clear();
}
void Downloader::downloadCloudSavesById(const std::string& product_id, int build_index)
{
auto res = this->cloudSaveListByIdForEach(product_id, build_index, [](cloudSaveFile &csf) {
iTotalRemainingBytes.fetch_add(csf.fileSize);
@ -4558,7 +4828,7 @@ void Downloader::downloadCloudSaves(const std::string& product_id, int build_ind
vThreads.push_back(std::thread(Downloader::processCloudSaveDownloadQueue, Globals::globalConfig, i));
}
this->printProgress(dlQueue);
this->printProgress(dlCloudSaveQueue);
// Join threads
for (unsigned int i = 0; i < vThreads.size(); ++i) {
@ -4597,25 +4867,6 @@ void Downloader::galaxyShowCloudSavesById(const std::string& product_id, int bui
});
}
void dirForEachHelper(const boost::filesystem::path &location, std::function<void(boost::filesystem::directory_iterator)> &f) {
boost::filesystem::directory_iterator begin { location };
boost::filesystem::directory_iterator end;
for(boost::filesystem::directory_iterator curr_dir { begin }; curr_dir != end; ++curr_dir) {
if(boost::filesystem::is_directory(*curr_dir)) {
dirForEachHelper(*curr_dir, f);
}
else {
f(curr_dir);
}
}
}
void dirForEach(const std::string &location, std::function<void(boost::filesystem::directory_iterator)> &&f) {
dirForEachHelper(location, f);
}
void Downloader::galaxyShowLocalCloudSavesById(const std::string& product_id, int build_index) {
auto name_to_locations = cloudSaveLocations(product_id, build_index);

15
src/util.cpp
View File

@ -8,6 +8,9 @@
#include <boost/filesystem.hpp>
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/copy.hpp>
#include <tinyxml2.h>
#include <json/json.h>
#include <fstream>
@ -765,7 +768,6 @@ void Util::CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf)
curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, conf.sUserAgent.c_str());
curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, conf.iTimeout);
curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
@ -882,3 +884,14 @@ curl_off_t Util::CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, c
return realsize;
}
curl_off_t Util::CurlReadChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, ChunkMemoryStruct *mem) {
curl_off_t realsize = std::min(size * nmemb, mem->size);
std::copy(mem->memory, mem->memory + realsize, (char*)contents);
mem->size -= realsize;
mem->memory += realsize;
return realsize;
}