diff --git a/include/galaxyapi.h b/include/galaxyapi.h index 31d518d..03b6e0b 100644 --- a/include/galaxyapi.h +++ b/include/galaxyapi.h @@ -64,6 +64,7 @@ class galaxyAPI Json::Value getDependenciesJson(); std::vector getFilteredDepotItemsVectorFromJson(const Json::Value& depot_json, const std::string& galaxy_language, const std::string& galaxy_arch, const bool& is_dependency = false); std::string getPathFromDownlinkUrl(const std::string& downlink_url, const std::string& gamename); + std::vector cdnUrlTemplatesFromJson(const Json::Value& json, const std::vector& cdnPriority); protected: private: CurlConfig curlConf; diff --git a/include/globalconstants.h b/include/globalconstants.h index 617bfc6..3e1be99 100644 --- a/include/globalconstants.h +++ b/include/globalconstants.h @@ -106,12 +106,16 @@ namespace GlobalConstants const unsigned int CDN_EDGECAST = 1 << 0; const unsigned int CDN_HIGHWINDS = 1 << 1; const unsigned int CDN_GOG = 1 << 2; + const unsigned int CDN_LUMEN = 1 << 3; + const unsigned int CDN_AKAMAI = 1 << 4; const std::vector GALAXY_CDNS = { { CDN_EDGECAST, "edgecast", "Edgecast", "ec|edgecast" }, { CDN_HIGHWINDS, "high_winds", "Highwinds", "hw|highwinds|high_winds" }, - { CDN_GOG, "gog_cdn", "GOG", "gog|gog_cdn" } + { CDN_GOG, "gog_cdn", "GOG", "gog|gog_cdn" }, + { CDN_LUMEN, "lumen", "Lumen", "lumen|lumen_cdn" }, + { CDN_AKAMAI, "akamai_edgecast_proxy", "Akamai", "akamai|akamai_cdn|akamai_ec|akamai_edgecast_proxy" } }; } diff --git a/main.cpp b/main.cpp index 7793011..e7edaba 100644 --- a/main.cpp +++ b/main.cpp @@ -276,7 +276,7 @@ int main(int argc, char *argv[]) ("galaxy-arch", bpo::value(&sGalaxyArch)->default_value("x64"), galaxy_arch_text.c_str()) ("galaxy-no-dependencies", bpo::value(&bNoGalaxyDependencies)->zero_tokens()->default_value(false), "Don't download dependencies during --galaxy-install") ("subdir-galaxy-install", bpo::value(&Globals::globalConfig.dirConf.sGalaxyInstallSubdir)->default_value("%install_dir%"), galaxy_install_subdir_text.c_str()) - ("galaxy-cdn-priority", bpo::value(&sGalaxyCDN)->default_value("edgecast,highwinds,gog_cdn"), galaxy_cdn_priority_text.c_str()) + ("galaxy-cdn-priority", bpo::value(&sGalaxyCDN)->default_value("edgecast,highwinds,akamai,lumen,gog_cdn"), galaxy_cdn_priority_text.c_str()) ("galaxy-delete-orphans", bpo::value(&Globals::globalConfig.dlConf.bGalaxyDeleteOrphans)->zero_tokens()->default_value(false), "Delete orphaned files during --galaxy-install") ; diff --git a/src/downloader.cpp b/src/downloader.cpp index 5035a96..159d06d 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -3645,8 +3645,13 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); galaxyDepotItem item; + std::string prev_product_id = ""; + std::vector cdnUrlTemplates; while (dlQueueGalaxy.try_pop(item)) { + if (item.product_id != prev_product_id) + cdnUrlTemplates.clear(); + vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); iTotalRemainingBytes.fetch_sub(item.totalSizeCompressed); @@ -3808,97 +3813,48 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con } } - Json::Value json; - if (item.isDependency) - json = galaxy->getDependencyLink(galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed)); - else - json = galaxy->getSecureLink(item.product_id, galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed)); - - if (json.empty()) + std::string galaxyPath = galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed); + // Get url templates for cdns + // Regular files can re-use these + // Dependencies require new url everytime + if (cdnUrlTemplates.empty() || item.isDependency) { - bChunkFailure = true; - std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")"; - msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix)); - break; - } + Json::Value json; + if (item.isDependency) + json = galaxy->getDependencyLink(galaxyPath); + else + json = galaxy->getSecureLink(item.product_id, "/"); - // Handle priority of CDNs - struct urlPriority - { - std::string url; - int priority; - }; - - // Build a vector of all urls and their priority score - std::vector cdnUrls; - for (unsigned int k = 0; k < json["urls"].size(); ++k) - { - std::string endpoint_name = json["urls"][k]["endpoint_name"].asString(); - - unsigned int score = conf.dlConf.vGalaxyCDNPriority.size(); - unsigned int cdn = Util::getOptionValue(endpoint_name, GlobalConstants::GALAXY_CDNS, false); - for (unsigned int idx = 0; idx < score; ++idx) + if (json.empty()) { - if (cdn & conf.dlConf.vGalaxyCDNPriority[idx]) - { - score = idx; - break; - } + bChunkFailure = true; + std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")"; + msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix)); + break; } - // Couldn't find a match when assigning score - if (score == conf.dlConf.vGalaxyCDNPriority.size()) - { - // Add index value to score - // This way unknown CDNs have priority based on the order they appear in json - score += k; - } - - // Build url according to url_format - std::string link_base_url = json["urls"][k]["parameters"]["base_url"].asString(); - std::string link_path = json["urls"][k]["parameters"]["path"].asString(); - std::string link_token = json["urls"][k]["parameters"]["token"].asString(); - - std::string url = json["urls"][k]["url_format"].asString(); - - while(Util::replaceString(url, "{base_url}", link_base_url)); - while(Util::replaceString(url, "{path}", link_path)); - while(Util::replaceString(url, "{token}", link_token)); - - // Highwinds specific - std::string link_hw_l= json["urls"][k]["parameters"]["l"].asString(); - std::string link_hw_source = json["urls"][k]["parameters"]["source"].asString(); - std::string link_hw_ttl = json["urls"][k]["parameters"]["ttl"].asString(); - std::string link_hw_gog_token = json["urls"][k]["parameters"]["gog_token"].asString(); - - while(Util::replaceString(url, "{l}", link_hw_l)); - while(Util::replaceString(url, "{source}", link_hw_source)); - while(Util::replaceString(url, "{ttl}", link_hw_ttl)); - while(Util::replaceString(url, "{gog_token}", link_hw_gog_token)); - - urlPriority cdnurl; - cdnurl.url = url; - cdnurl.priority = score; - cdnUrls.push_back(cdnurl); + cdnUrlTemplates = galaxy->cdnUrlTemplatesFromJson(json, conf.dlConf.vGalaxyCDNPriority); } - if (cdnUrls.empty()) + if (cdnUrlTemplates.empty()) { bChunkFailure = true; msgQueue.push(Message(path.string() + ": Failed to get download url", MSGTYPE_ERROR, msg_prefix)); break; } - // Sort urls by priority (lowest score first) - std::sort(cdnUrls.begin(), cdnUrls.end(), - [](urlPriority a, urlPriority b) - { - return (a.priority < b.priority); - } - ); - - // Select url with lowest priority score - std::string url = cdnUrls[0].url; + std::string url = cdnUrlTemplates[0]; + if (item.isDependency) + { + while(Util::replaceString(url, "{LGOGDOWNLOADER_GALAXY_PATH}", "")); + cdnUrlTemplates.clear(); // Clear templates + } + else + { + galaxyPath = "/" + galaxyPath; + while(Util::replaceString(url, "{LGOGDOWNLOADER_GALAXY_PATH}", galaxyPath)); + prev_product_id = item.product_id; + } ChunkMemoryStruct chunk; chunk.memory = (char *) malloc(1); diff --git a/src/galaxyapi.cpp b/src/galaxyapi.cpp index 42fa438..17595e5 100644 --- a/src/galaxyapi.cpp +++ b/src/galaxyapi.cpp @@ -540,3 +540,93 @@ std::string galaxyAPI::getPathFromDownlinkUrl(const std::string& downlink_url, c return path; } + +std::vector galaxyAPI::cdnUrlTemplatesFromJson(const Json::Value& json, const std::vector& cdnPriority) +{ + // Handle priority of CDNs + struct urlPriority + { + std::string url; + int priority; + }; + std::vector cdnUrls; + + // Build a vector of all urls and their priority score + for (unsigned int i = 0; i < json["urls"].size(); ++i) + { + std::string endpoint_name = json["urls"][i]["endpoint_name"].asString(); + + unsigned int score = cdnPriority.size(); + unsigned int cdn = Util::getOptionValue(endpoint_name, GlobalConstants::GALAXY_CDNS, false); + for (unsigned int idx = 0; idx < score; ++idx) + { + if (cdn & cdnPriority[idx]) + { + score = idx; + break; + } + } + + // Couldn't find a match when assigning score + if (score == cdnPriority.size()) + { + // Add index value to score + // This way unknown CDNs have priority based on the order they appear in json + score += i; + } + + // Build url according to url_format + std::string link_base_url = json["urls"][i]["parameters"]["base_url"].asString(); + std::string link_path = json["urls"][i]["parameters"]["path"].asString(); + std::string link_token = json["urls"][i]["parameters"]["token"].asString(); + + // Add our own template to path + link_path += "{LGOGDOWNLOADER_GALAXY_PATH}"; + + std::string url = json["urls"][i]["url_format"].asString(); + + while(Util::replaceString(url, "{base_url}", link_base_url)); + while(Util::replaceString(url, "{path}", link_path)); + while(Util::replaceString(url, "{token}", link_token)); + + // Highwinds specific + std::string link_hw_l= json["urls"][i]["parameters"]["l"].asString(); + std::string link_hw_source = json["urls"][i]["parameters"]["source"].asString(); + std::string link_hw_ttl = json["urls"][i]["parameters"]["ttl"].asString(); + std::string link_hw_gog_token = json["urls"][i]["parameters"]["gog_token"].asString(); + + while(Util::replaceString(url, "{l}", link_hw_l)); + while(Util::replaceString(url, "{source}", link_hw_source)); + while(Util::replaceString(url, "{ttl}", link_hw_ttl)); + while(Util::replaceString(url, "{gog_token}", link_hw_gog_token)); + + // Lumen specific + std::string dirs = json["urls"][i]["parameters"]["dirs"].asString(); + std::string expires_at = json["urls"][i]["parameters"]["expires_at"].asString(); + + while(Util::replaceString(url, "{dirs}", dirs)); + while(Util::replaceString(url, "{expires_at}", expires_at)); + + urlPriority cdnurl; + cdnurl.url = url; + cdnurl.priority = score; + cdnUrls.push_back(cdnurl); + } + + if (!cdnUrls.empty()) + { + // Sort urls by priority (lowest score first) + std::sort(cdnUrls.begin(), cdnUrls.end(), + [](urlPriority a, urlPriority b) + { + return (a.priority < b.priority); + } + ); + } + + std::vector cdnUrlTemplates; + for (auto cdnurl : cdnUrls) + cdnUrlTemplates.push_back(cdnurl.url); + + return cdnUrlTemplates; +} diff --git a/src/util.cpp b/src/util.cpp index 7650b21..5fd1c35 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -577,7 +577,8 @@ unsigned int Util::getOptionValue(const std::string& str, const std::vector