Mirrors/lgogdownloader
2
0
Fork 0
mirror of https://github.com/Sude-/lgogdownloader.git synced 2024-11-20 03:39:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

Reduce the amount of request to cdn with --galaxy-install

Browse Source 
Should fix the issue of getting temporarily blocked by GOG cdn when installing game with lots of small files
This commit is contained in:
Sude 2022-02-17 10:30:38 +02:00
parent 30f698867c
commit 983592d09e
6 changed files with 133 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/galaxyapi.h
View File

@ -64,6 +64,7 @@ class galaxyAPI
Json::Value getDependenciesJson();
std::vector<galaxyDepotItem> getFilteredDepotItemsVectorFromJson(const Json::Value& depot_json, const std::string& galaxy_language, const std::string& galaxy_arch, const bool& is_dependency = false);
std::string getPathFromDownlinkUrl(const std::string& downlink_url, const std::string& gamename);
std::vector<std::string> cdnUrlTemplatesFromJson(const Json::Value& json, const std::vector<unsigned int>& cdnPriority);
protected:
private:
CurlConfig curlConf;

6
include/globalconstants.h
View File

@ -106,12 +106,16 @@ namespace GlobalConstants
const unsigned int CDN_EDGECAST = 1 << 0;
const unsigned int CDN_HIGHWINDS = 1 << 1;
const unsigned int CDN_GOG = 1 << 2;
const unsigned int CDN_LUMEN = 1 << 3;
const unsigned int CDN_AKAMAI = 1 << 4;
const std::vector<optionsStruct> GALAXY_CDNS =
{
{ CDN_EDGECAST, "edgecast", "Edgecast", "ec|edgecast" },
{ CDN_HIGHWINDS, "high_winds", "Highwinds", "hw|highwinds|high_winds" },
{ CDN_GOG, "gog_cdn", "GOG", "gog|gog_cdn" }
{ CDN_GOG, "gog_cdn", "GOG", "gog|gog_cdn" },
{ CDN_LUMEN, "lumen", "Lumen", "lumen|lumen_cdn" },
{ CDN_AKAMAI, "akamai_edgecast_proxy", "Akamai", "akamai|akamai_cdn|akamai_ec|akamai_edgecast_proxy" }
};
}

2
main.cpp
View File

@ -276,7 +276,7 @@ int main(int argc, char *argv[])
("galaxy-arch", bpo::value<std::string>(&sGalaxyArch)->default_value("x64"), galaxy_arch_text.c_str())
("galaxy-no-dependencies", bpo::value<bool>(&bNoGalaxyDependencies)->zero_tokens()->default_value(false), "Don't download dependencies during --galaxy-install")
("subdir-galaxy-install", bpo::value<std::string>(&Globals::globalConfig.dirConf.sGalaxyInstallSubdir)->default_value("%install_dir%"), galaxy_install_subdir_text.c_str())
("galaxy-cdn-priority", bpo::value<std::string>(&sGalaxyCDN)->default_value("edgecast,highwinds,gog_cdn"), galaxy_cdn_priority_text.c_str())
("galaxy-cdn-priority", bpo::value<std::string>(&sGalaxyCDN)->default_value("edgecast,highwinds,akamai,lumen,gog_cdn"), galaxy_cdn_priority_text.c_str())
("galaxy-delete-orphans", bpo::value<bool>(&Globals::globalConfig.dlConf.bGalaxyDeleteOrphans)->zero_tokens()->default_value(false), "Delete orphaned files during --galaxy-install")
;

112
src/downloader.cpp
View File

@ -3645,8 +3645,13 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo);
galaxyDepotItem item;
std::string prev_product_id = "";
std::vector<std::string> cdnUrlTemplates;
while (dlQueueGalaxy.try_pop(item))
{
if (item.product_id != prev_product_id)
cdnUrlTemplates.clear();
vDownloadInfo[tid].setStatus(DLSTATUS_STARTING);
iTotalRemainingBytes.fetch_sub(item.totalSizeCompressed);
@ -3808,97 +3813,48 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con
}
}
Json::Value json;
if (item.isDependency)
json = galaxy->getDependencyLink(galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed));
else
json = galaxy->getSecureLink(item.product_id, galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed));
if (json.empty())
std::string galaxyPath = galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed);
// Get url templates for cdns
// Regular files can re-use these
// Dependencies require new url everytime
if (cdnUrlTemplates.empty() || item.isDependency)
{
bChunkFailure = true;
std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")";
msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix));
break;
}
Json::Value json;
if (item.isDependency)
json = galaxy->getDependencyLink(galaxyPath);
else
json = galaxy->getSecureLink(item.product_id, "/");
// Handle priority of CDNs
struct urlPriority
{
std::string url;
int priority;
};
// Build a vector of all urls and their priority score
std::vector<urlPriority> cdnUrls;
for (unsigned int k = 0; k < json["urls"].size(); ++k)
{
std::string endpoint_name = json["urls"][k]["endpoint_name"].asString();
unsigned int score = conf.dlConf.vGalaxyCDNPriority.size();
unsigned int cdn = Util::getOptionValue(endpoint_name, GlobalConstants::GALAXY_CDNS, false);
for (unsigned int idx = 0; idx < score; ++idx)
if (json.empty())
{
if (cdn & conf.dlConf.vGalaxyCDNPriority[idx])
{
score = idx;
break;
}
bChunkFailure = true;
std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")";
msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix));
break;
}
// Couldn't find a match when assigning score
if (score == conf.dlConf.vGalaxyCDNPriority.size())
{
// Add index value to score
// This way unknown CDNs have priority based on the order they appear in json
score += k;
}
// Build url according to url_format
std::string link_base_url = json["urls"][k]["parameters"]["base_url"].asString();
std::string link_path = json["urls"][k]["parameters"]["path"].asString();
std::string link_token = json["urls"][k]["parameters"]["token"].asString();
std::string url = json["urls"][k]["url_format"].asString();
while(Util::replaceString(url, "{base_url}", link_base_url));
while(Util::replaceString(url, "{path}", link_path));
while(Util::replaceString(url, "{token}", link_token));
// Highwinds specific
std::string link_hw_l= json["urls"][k]["parameters"]["l"].asString();
std::string link_hw_source = json["urls"][k]["parameters"]["source"].asString();
std::string link_hw_ttl = json["urls"][k]["parameters"]["ttl"].asString();
std::string link_hw_gog_token = json["urls"][k]["parameters"]["gog_token"].asString();
while(Util::replaceString(url, "{l}", link_hw_l));
while(Util::replaceString(url, "{source}", link_hw_source));
while(Util::replaceString(url, "{ttl}", link_hw_ttl));
while(Util::replaceString(url, "{gog_token}", link_hw_gog_token));
urlPriority cdnurl;
cdnurl.url = url;
cdnurl.priority = score;
cdnUrls.push_back(cdnurl);
cdnUrlTemplates = galaxy->cdnUrlTemplatesFromJson(json, conf.dlConf.vGalaxyCDNPriority);
}
if (cdnUrls.empty())
if (cdnUrlTemplates.empty())
{
bChunkFailure = true;
msgQueue.push(Message(path.string() + ": Failed to get download url", MSGTYPE_ERROR, msg_prefix));
break;
}
// Sort urls by priority (lowest score first)
std::sort(cdnUrls.begin(), cdnUrls.end(),
[](urlPriority a, urlPriority b)
{
return (a.priority < b.priority);
}
);
// Select url with lowest priority score
std::string url = cdnUrls[0].url;
std::string url = cdnUrlTemplates[0];
if (item.isDependency)
{
while(Util::replaceString(url, "{LGOGDOWNLOADER_GALAXY_PATH}", ""));
cdnUrlTemplates.clear(); // Clear templates
}
else
{
galaxyPath = "/" + galaxyPath;
while(Util::replaceString(url, "{LGOGDOWNLOADER_GALAXY_PATH}", galaxyPath));
prev_product_id = item.product_id;
}
ChunkMemoryStruct chunk;
chunk.memory = (char *) malloc(1);

90
src/galaxyapi.cpp
View File

@ -540,3 +540,93 @@ std::string galaxyAPI::getPathFromDownlinkUrl(const std::string& downlink_url, c
return path;
}
std::vector<std::string> galaxyAPI::cdnUrlTemplatesFromJson(const Json::Value& json, const std::vector<unsigned int>& cdnPriority)
{
// Handle priority of CDNs
struct urlPriority
{
std::string url;
int priority;
};
std::vector<urlPriority> cdnUrls;
// Build a vector of all urls and their priority score
for (unsigned int i = 0; i < json["urls"].size(); ++i)
{
std::string endpoint_name = json["urls"][i]["endpoint_name"].asString();
unsigned int score = cdnPriority.size();
unsigned int cdn = Util::getOptionValue(endpoint_name, GlobalConstants::GALAXY_CDNS, false);
for (unsigned int idx = 0; idx < score; ++idx)
{
if (cdn & cdnPriority[idx])
{
score = idx;
break;
}
}
// Couldn't find a match when assigning score
if (score == cdnPriority.size())
{
// Add index value to score
// This way unknown CDNs have priority based on the order they appear in json
score += i;
}
// Build url according to url_format
std::string link_base_url = json["urls"][i]["parameters"]["base_url"].asString();
std::string link_path = json["urls"][i]["parameters"]["path"].asString();
std::string link_token = json["urls"][i]["parameters"]["token"].asString();
// Add our own template to path
link_path += "{LGOGDOWNLOADER_GALAXY_PATH}";
std::string url = json["urls"][i]["url_format"].asString();
while(Util::replaceString(url, "{base_url}", link_base_url));
while(Util::replaceString(url, "{path}", link_path));
while(Util::replaceString(url, "{token}", link_token));
// Highwinds specific
std::string link_hw_l= json["urls"][i]["parameters"]["l"].asString();
std::string link_hw_source = json["urls"][i]["parameters"]["source"].asString();
std::string link_hw_ttl = json["urls"][i]["parameters"]["ttl"].asString();
std::string link_hw_gog_token = json["urls"][i]["parameters"]["gog_token"].asString();
while(Util::replaceString(url, "{l}", link_hw_l));
while(Util::replaceString(url, "{source}", link_hw_source));
while(Util::replaceString(url, "{ttl}", link_hw_ttl));
while(Util::replaceString(url, "{gog_token}", link_hw_gog_token));
// Lumen specific
std::string dirs = json["urls"][i]["parameters"]["dirs"].asString();
std::string expires_at = json["urls"][i]["parameters"]["expires_at"].asString();
while(Util::replaceString(url, "{dirs}", dirs));
while(Util::replaceString(url, "{expires_at}", expires_at));
urlPriority cdnurl;
cdnurl.url = url;
cdnurl.priority = score;
cdnUrls.push_back(cdnurl);
}
if (!cdnUrls.empty())
{
// Sort urls by priority (lowest score first)
std::sort(cdnUrls.begin(), cdnUrls.end(),
[](urlPriority a, urlPriority b)
{
return (a.priority < b.priority);
}
);
}
std::vector<std::string> cdnUrlTemplates;
for (auto cdnurl : cdnUrls)
cdnUrlTemplates.push_back(cdnurl.url);
return cdnUrlTemplates;
}

3
src/util.cpp
View File

@ -577,7 +577,8 @@ unsigned int Util::getOptionValue(const std::string& str, const std::vector<Glob
break;
}
}
else if (str == options[i].code)
if (str == options[i].code)
{
value = options[i].id;
break;