From ab84d6f82720b8ca24704dae08673d8741cfede6 Mon Sep 17 00:00:00 2001 From: Sude Date: Wed, 12 Sep 2018 20:07:59 +0300 Subject: [PATCH] Galaxy: Add option to set CDN priority --galaxy-cdn-priority option allows to set priorities for content delivery networks --- include/config.h | 2 ++ include/globalconstants.h | 12 +++++++ main.cpp | 18 ++++++++-- src/downloader.cpp | 73 +++++++++++++++++++++++++++++++-------- 4 files changed, 88 insertions(+), 17 deletions(-) diff --git a/include/config.h b/include/config.h index 8205405..f038229 100644 --- a/include/config.h +++ b/include/config.h @@ -32,8 +32,10 @@ struct DownloadConfig { unsigned int iInstallerPlatform; unsigned int iInstallerLanguage; + unsigned int iGalaxyCDN; std::vector vPlatformPriority; std::vector vLanguagePriority; + std::vector vGalaxyCDNPriority; unsigned int iInclude; unsigned int iGalaxyPlatform; unsigned int iGalaxyLanguage; diff --git a/include/globalconstants.h b/include/globalconstants.h index 55e899f..788086b 100644 --- a/include/globalconstants.h +++ b/include/globalconstants.h @@ -93,6 +93,18 @@ namespace GlobalConstants { ARCH_X86, "32", "32-bit", "32|x86|32bit|32-bit" }, { ARCH_X64, "64", "64-bit", "64|x64|64bit|64-bit" } }; + + // Galaxy CDNs + const unsigned int CDN_EDGECAST = 1 << 0; + const unsigned int CDN_HIGHWINDS = 1 << 1; + const unsigned int CDN_GOG = 1 << 2; + + const std::vector GALAXY_CDNS = + { + { CDN_EDGECAST, "edgecast", "Edgecast", "ec|edgecast" }, + { CDN_HIGHWINDS, "high_winds", "Highwinds", "hw|highwinds|high_winds" }, + { CDN_GOG, "gog_cdn", "GOG", "gog|gog_cdn" } + }; } #endif // GLOBALCONSTANTS_H_INCLUDED diff --git a/main.cpp b/main.cpp index 1305f4b..2df784d 100644 --- a/main.cpp +++ b/main.cpp @@ -83,7 +83,7 @@ int main(int argc, char *argv[]) std::string language_text = "Select which language installers are downloaded\n"; for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i) { - language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n"; + language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n"; } language_text += "All = all"; language_text += "\n\n" + priority_help_text; @@ -93,14 +93,14 @@ int main(int argc, char *argv[]) std::string galaxy_language_text = "Select language\n"; for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i) { - galaxy_language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n"; + galaxy_language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n"; } // Create help text for --galaxy-arch option std::string galaxy_arch_text = "Select architecture\n"; for (unsigned int i = 0; i < GlobalConstants::GALAXY_ARCHS.size(); ++i) { - galaxy_arch_text += GlobalConstants::GALAXY_ARCHS[i].str + " = " + GlobalConstants::GALAXY_ARCHS[i].regexp + "\n"; + galaxy_arch_text += GlobalConstants::GALAXY_ARCHS[i].str + " = " + GlobalConstants::GALAXY_ARCHS[i].regexp + "\n"; } // Create help text for --subdir-galaxy-install option @@ -117,6 +117,14 @@ int main(int argc, char *argv[]) "> space\n" "> - _ . ( ) [ ] { }"; + // Create help text for --galaxy-cdn-priority option + std::string galaxy_cdn_priority_text = "Set priority for used CDNs\n"; + for (unsigned int i = 0; i < GlobalConstants::GALAXY_CDNS.size(); ++i) + { + galaxy_cdn_priority_text += GlobalConstants::GALAXY_CDNS[i].str + " = " + GlobalConstants::GALAXY_CDNS[i].regexp + "\n"; + } + galaxy_cdn_priority_text += "\n" + priority_help_text; + // Create help text for --check-orphans std::string orphans_regex_default = ".*\\.(zip|exe|bin|dmg|old|deb|tar\\.gz|pkg|sh)$"; // Limit to files with these extensions (".old" is for renamed older version files) std::string check_orphans_text = "Check for orphaned files (files found on local filesystem that are not found on GOG servers). Sets regular expression filter (Perl syntax) for files to check. If no argument is given then the regex defaults to '" + orphans_regex_default + "'"; @@ -166,6 +174,7 @@ int main(int argc, char *argv[]) std::string sGalaxyPlatform; std::string sGalaxyLanguage; std::string sGalaxyArch; + std::string sGalaxyCDN; Globals::globalConfig.bReport = false; // Commandline options (no config file) options_cli_no_cfg.add_options() @@ -255,6 +264,7 @@ int main(int argc, char *argv[]) ("galaxy-arch", bpo::value(&sGalaxyArch)->default_value("x64"), galaxy_arch_text.c_str()) ("galaxy-no-dependencies", bpo::value(&bNoGalaxyDependencies)->zero_tokens()->default_value(false), "Don't download dependencies during --galaxy-install") ("subdir-galaxy-install", bpo::value(&Globals::globalConfig.dirConf.sGalaxyInstallSubdir)->default_value("%install_dir%"), galaxy_install_subdir_text.c_str()) + ("galaxy-cdn-priority", bpo::value(&sGalaxyCDN)->default_value("edgecast,highwinds,gog_cdn"), galaxy_cdn_priority_text.c_str()) ; options_cli_all.add(options_cli_no_cfg).add(options_cli_cfg).add(options_cli_experimental); @@ -493,6 +503,8 @@ int main(int argc, char *argv[]) if (Globals::globalConfig.dlConf.iGalaxyArch == 0 || Globals::globalConfig.dlConf.iGalaxyArch == Util::getOptionValue("all", GlobalConstants::GALAXY_ARCHS, false)) Globals::globalConfig.dlConf.iGalaxyArch = GlobalConstants::ARCH_X64; + Util::parseOptionString(sGalaxyCDN, Globals::globalConfig.dlConf.vGalaxyCDNPriority, Globals::globalConfig.dlConf.iGalaxyCDN, GlobalConstants::GALAXY_CDNS); + unsigned int include_value = 0; unsigned int exclude_value = 0; std::vector vInclude = Util::tokenize(sIncludeOptions, ","); diff --git a/src/downloader.cpp b/src/downloader.cpp index e31092f..d554071 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -3636,31 +3636,76 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con else json = galaxy->getSecureLink(item.product_id, galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed)); - // Prefer edgecast urls - bool bPreferEdgecast = true; - unsigned int idx = 0; + // Handle priority of CDNs + struct urlPriority + { + std::string url; + int priority; + }; + + // Build a vector of all urls and their priority score + std::vector cdnUrls; for (unsigned int k = 0; k < json["urls"].size(); ++k) { std::string endpoint_name = json["urls"][k]["endpoint_name"].asString(); - if (bPreferEdgecast) + + unsigned int score = conf.dlConf.vGalaxyCDNPriority.size(); + unsigned int cdn = Util::getOptionValue(endpoint_name, GlobalConstants::GALAXY_CDNS, false); + for (unsigned int idx = 0; idx < score; ++idx) { - if (endpoint_name == "edgecast") + if (cdn & conf.dlConf.vGalaxyCDNPriority[idx]) { - idx = k; + score = idx; break; } } + + // Couldn't find a match when assigning score + if (score == conf.dlConf.vGalaxyCDNPriority.size()) + { + // Add index value to score + // This way unknown CDNs have priority based on the order they appear in json + score += k; + } + + // Build url according to url_format + std::string link_base_url = json["urls"][k]["parameters"]["base_url"].asString(); + std::string link_path = json["urls"][k]["parameters"]["path"].asString(); + std::string link_token = json["urls"][k]["parameters"]["token"].asString(); + + std::string url = json["urls"][k]["url_format"].asString(); + + while(Util::replaceString(url, "{base_url}", link_base_url)); + while(Util::replaceString(url, "{path}", link_path)); + while(Util::replaceString(url, "{token}", link_token)); + + // Highwinds specific + std::string link_hw_l= json["urls"][k]["parameters"]["l"].asString(); + std::string link_hw_source = json["urls"][k]["parameters"]["source"].asString(); + std::string link_hw_ttl = json["urls"][k]["parameters"]["ttl"].asString(); + std::string link_hw_gog_token = json["urls"][k]["parameters"]["gog_token"].asString(); + + while(Util::replaceString(url, "{l}", link_hw_l)); + while(Util::replaceString(url, "{source}", link_hw_source)); + while(Util::replaceString(url, "{ttl}", link_hw_ttl)); + while(Util::replaceString(url, "{gog_token}", link_hw_gog_token)); + + urlPriority cdnurl; + cdnurl.url = url; + cdnurl.priority = score; + cdnUrls.push_back(cdnurl); } - // Build url according to url_format - std::string link_base_url = json["urls"][idx]["parameters"]["base_url"].asString(); - std::string link_path = json["urls"][idx]["parameters"]["path"].asString(); - std::string link_token = json["urls"][idx]["parameters"]["token"].asString(); - std::string url = json["urls"][idx]["url_format"].asString(); + // Sort urls by priority (lowest score first) + std::sort(cdnUrls.begin(), cdnUrls.end(), + [](urlPriority a, urlPriority b) + { + return (a.priority < b.priority); + } + ); - while(Util::replaceString(url, "{base_url}", link_base_url)); - while(Util::replaceString(url, "{path}", link_path)); - while(Util::replaceString(url, "{token}", link_token)); + // Select url with lowest priority score + std::string url = cdnUrls[0].url; curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0);