diff --git a/include/config.h b/include/config.h index e5583c1..3743480 100644 --- a/include/config.h +++ b/include/config.h @@ -309,15 +309,12 @@ class Config // Lists Blacklist blacklist; Blacklist ignorelist; - Blacklist gamehasdlc; // Cloud save options std::vector cloudWhiteList; std::vector cloudBlackList; bool bCloudForce; - std::string sGameHasDLCList; - // Integers int iRetries; unsigned int iThreads; diff --git a/main.cpp b/main.cpp index 6a3b539..f52446e 100644 --- a/main.cpp +++ b/main.cpp @@ -59,7 +59,6 @@ int main(int argc, char *argv[]) Globals::globalConfig.sConfigFilePath = Globals::globalConfig.sConfigDirectory + "/config.cfg"; Globals::globalConfig.sBlacklistFilePath = Globals::globalConfig.sConfigDirectory + "/blacklist.txt"; Globals::globalConfig.sIgnorelistFilePath = Globals::globalConfig.sConfigDirectory + "/ignorelist.txt"; - Globals::globalConfig.sGameHasDLCListFilePath = Globals::globalConfig.sConfigDirectory + "/game_has_dlc.txt"; Globals::globalConfig.sTransformConfigFilePath = Globals::globalConfig.sConfigDirectory + "/transformations.json"; Globals::galaxyConf.setFilepath(Globals::globalConfig.sConfigDirectory + "/galaxy_tokens.json"); @@ -196,7 +195,6 @@ int main(int argc, char *argv[]) bool bNoSubDirectories = false; bool bNoPlatformDetection = false; bool bNoGalaxyDependencies = false; - bool bUseDLCList = false; bool bNoFastStatusCheck = false; std::string sInstallerPlatform; std::string sInstallerLanguage; @@ -290,8 +288,6 @@ int main(int argc, char *argv[]) ("save-changelogs", bpo::value(&Globals::globalConfig.dlConf.bSaveChangelogs)->zero_tokens()->default_value(false), "Save changelogs when downloading") ("threads", bpo::value(&Globals::globalConfig.iThreads)->default_value(4), "Number of download threads") ("info-threads", bpo::value(&Globals::globalConfig.iInfoThreads)->default_value(4), "Number of threads for getting product info") - ("use-dlc-list", bpo::value(&bUseDLCList)->zero_tokens()->default_value(false), "Use DLC list specified with --dlc-list") - ("dlc-list", bpo::value(&Globals::globalConfig.sGameHasDLCList)->default_value("https://raw.githubusercontent.com/Sude-/lgogdownloader-lists/master/game_has_dlc.txt"), "Set URL for list of games that have DLC") ("progress-interval", bpo::value(&Globals::globalConfig.iProgressInterval)->default_value(100), "Set interval for progress bar update (milliseconds)\nValue must be between 1 and 10000") ("lowspeed-timeout", bpo::value(&Globals::globalConfig.curlConf.iLowSpeedTimeout)->default_value(30), "Set time in number seconds that the transfer speed should be below the rate set with --lowspeed-rate for it to considered too slow and aborted") ("lowspeed-rate", bpo::value(&Globals::globalConfig.curlConf.iLowSpeedTimeoutRate)->default_value(200), "Set average transfer speed in bytes per second that the transfer should be below during time specified with --lowspeed-timeout for it to be considered too slow and aborted") @@ -449,33 +445,6 @@ int main(int argc, char *argv[]) Globals::globalConfig.transformationsJSON = Util::readJsonFile(Globals::globalConfig.sTransformConfigFilePath); } - if (!bUseDLCList) - Globals::globalConfig.sGameHasDLCList = ""; - - if (Globals::globalConfig.sIgnoreDLCCountRegex.empty()) - { - if (boost::filesystem::exists(Globals::globalConfig.sGameHasDLCListFilePath) && bUseDLCList) - { - std::ifstream ifs(Globals::globalConfig.sGameHasDLCListFilePath.c_str()); - if (!ifs) - { - std::cerr << "Could not open list of games that have dlc: " << Globals::globalConfig.sGameHasDLCListFilePath << std::endl; - return 1; - } - else - { - std::string line; - std::vector lines; - while (!ifs.eof()) - { - std::getline(ifs, line); - lines.push_back(std::move(line)); - } - Globals::globalConfig.gamehasdlc.initialize(lines); - } - } - } - #ifdef USE_QT_GUI_LOGIN if (Globals::globalConfig.bForceGUILogin) { diff --git a/man/lgogdownloader.1 b/man/lgogdownloader.1 index b74d43a..2e79b39 100644 --- a/man/lgogdownloader.1 +++ b/man/lgogdownloader.1 @@ -411,12 +411,6 @@ Number of download threads \fB\-\-info\-threads\fR arg (=4) Number of threads for getting product info .TP -\fB\-\-use\-dlc\-list\fR -Use DLC list specified with \fB\-\-dlc\-list\fR -.TP -\fB\-\-dlc\-list\fR arg (=https://raw.githubusercontent.com/Sude\-/lgogdownloader\-lists/master/game_has_dlc.txt) -Set URL for list of games that have DLC -.TP \fB\-\-progress\-interval\fR arg (=100) Set interval for progress bar update (milliseconds) .br @@ -742,17 +736,6 @@ blacklist. .br It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader. -.TP -\fI$XDG_CONFIG_HOME/lgogdownloader/game_has_dlc.txt\fP -Allows user to specify which games have dlc and should have their DLC count -information ignored. The file has the same format and interpretation as a -blacklist. -.br -It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader. -.br -If the file exists lgogdownloader uses it instead of list specified with -\fB--dlc-list\fP option - .TP \fI$XDG_CONFIG_HOME/lgogdownloader/gamespecific/gamename.conf\fP JSON formatted file. Sets game specific settings for \fBgamename\fP. diff --git a/src/downloader.cpp b/src/downloader.cpp index 20d91d4..dfdba94 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -200,16 +200,6 @@ bool Downloader::isLoggedIn() */ int Downloader::init() { - if (!Globals::globalConfig.sGameHasDLCList.empty()) - { - if (Globals::globalConfig.gamehasdlc.empty()) - { - std::string game_has_dlc_list = this->getResponse(Globals::globalConfig.sGameHasDLCList); - if (!game_has_dlc_list.empty()) - Globals::globalConfig.gamehasdlc.initialize(Util::tokenize(game_has_dlc_list, "\n")); - } - } - if (!gogGalaxy->init()) { if (gogGalaxy->refreshLogin()) diff --git a/src/util.cpp b/src/util.cpp index 47a4b8a..d88a602 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -498,6 +498,8 @@ std::vector Util::getDLCNamesFromJSON(const Json::Value &root) { std::string gamename; std::string url_prefix = "/downloads/"; + if (urls[i].find(url_prefix) == std::string::npos) + continue; gamename.assign(urls[i].begin()+urls[i].find(url_prefix)+url_prefix.length(), urls[i].begin()+urls[i].find_last_of("/")); bool bDuplicate = false; diff --git a/src/website.cpp b/src/website.cpp index 5aa42cd..b272ea7 100644 --- a/src/website.cpp +++ b/src/website.cpp @@ -97,7 +97,7 @@ std::vector Website::getGames() { std::vector games; Json::Value root; - int i = 1; + int iPage = 1; bool bAllPagesParsed = false; int iUpdated = Globals::globalConfig.bUpdated ? 1 : 0; int iHidden = 0; @@ -111,9 +111,10 @@ std::vector Website::getGames() } Globals::vOwnedGamesIds = this->getOwnedGamesIds(); + std::vector jsonProductInfo; do { - std::string url = "https://www.gog.com/account/getFilteredProducts?hiddenFlag=" + std::to_string(iHidden) + "&isUpdated=" + std::to_string(iUpdated) + "&mediaType=1&sortBy=title&system=&page=" + std::to_string(i); + std::string url = "https://www.gog.com/account/getFilteredProducts?hiddenFlag=" + std::to_string(iHidden) + "&isUpdated=" + std::to_string(iUpdated) + "&mediaType=1&sortBy=title&system=&page=" + std::to_string(iPage); if (!tags.empty()) url += "&tags=" + tags; @@ -121,6 +122,8 @@ std::vector Website::getGames() if (root.empty()) continue; + std::cerr << "\033[KGetting product data " << root["page"].asInt() << " / " << root["totalPages"].asInt() << "\r" << std::flush; + if (root["page"].asInt() == root["totalPages"].asInt() || root["totalPages"].asInt() == 0) bAllPagesParsed = true; @@ -129,124 +132,124 @@ std::vector Website::getGames() { bAllPagesParsed = false; iHidden = 1; - i = 0; // Set to 0 because we increment it at the end of the loop + iPage = 0; // Set to 0 because we increment it at the end of the loop } if (root["products"].isArray()) { - for (unsigned int i = 0; i < root["products"].size(); ++i) + for (auto product : root["products"]) + jsonProductInfo.push_back(product); + } + iPage++; + } while (!bAllPagesParsed); + std::cerr << std::endl; + + unsigned int iProduct = 0; + unsigned int iProductTotal = jsonProductInfo.size(); + for (auto product : jsonProductInfo) + { + iProduct++; + std::cerr << "\033[KGetting game names " << iProduct << " / " << iProductTotal << "\r" << std::flush; + gameItem game; + game.name = product["slug"].asString(); + game.id = product["id"].isInt() ? std::to_string(product["id"].asInt()) : product["id"].asString(); + game.isnew = product["isNew"].asBool(); + + if (product.isMember("updates")) + { + if (product["updates"].isNull()) { - std::cerr << "\033[KGetting game names " << "(" << root["page"].asInt() << "/" << root["totalPages"].asInt() << ") " << i+1 << " / " << root["products"].size() << "\r" << std::flush; - Json::Value product = root["products"][i]; - gameItem game; - game.name = product["slug"].asString(); - game.id = product["id"].isInt() ? std::to_string(product["id"].asInt()) : product["id"].asString(); - game.isnew = product["isNew"].asBool(); - - if (product.isMember("updates")) + /* In some cases the value can be null. + * For example when user owns a dlc but not the base game + * https://github.com/Sude-/lgogdownloader/issues/101 + * Assume that there are no updates in this case */ + game.updates = 0; + } + else if (product["updates"].isInt()) + game.updates = product["updates"].asInt(); + else + { + try { - if (product["updates"].isNull()) - { - /* In some cases the value can be null. - * For example when user owns a dlc but not the base game - * https://github.com/Sude-/lgogdownloader/issues/101 - * Assume that there are no updates in this case */ - game.updates = 0; - } - else if (product["updates"].isInt()) - game.updates = product["updates"].asInt(); - else - { - try - { - game.updates = std::stoi(product["updates"].asString()); - } - catch (std::invalid_argument& e) - { - game.updates = 0; // Assume no updates - } - } + game.updates = std::stoi(product["updates"].asString()); } - - unsigned int platform = 0; - if (product["worksOn"]["Windows"].asBool()) - platform |= GlobalConstants::PLATFORM_WINDOWS; - if (product["worksOn"]["Mac"].asBool()) - platform |= GlobalConstants::PLATFORM_MAC; - if (product["worksOn"]["Linux"].asBool()) - platform |= GlobalConstants::PLATFORM_LINUX; - - // Skip if not new and flag is set - if (Globals::globalConfig.bNew && !game.isnew) - continue; - - // Skip if platform doesn't match - if (Globals::globalConfig.bPlatformDetection && !(platform & Globals::globalConfig.dlConf.iInstallerPlatform)) - continue; - - // Filter the game list - if (!Globals::globalConfig.sGameRegex.empty()) + catch (std::invalid_argument& e) { - boost::regex expression(Globals::globalConfig.sGameRegex); - boost::match_results what; - if (!boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex - continue; + game.updates = 0; // Assume no updates } - - if (Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC) - { - int dlcCount = product["dlcCount"].asInt(); - - bool bDownloadDLCInfo = (dlcCount != 0); - - if (!bDownloadDLCInfo && !Globals::globalConfig.sIgnoreDLCCountRegex.empty()) - { - boost::regex expression(Globals::globalConfig.sIgnoreDLCCountRegex); - boost::match_results what; - if (boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex - { - bDownloadDLCInfo = true; - } - } - - if (!bDownloadDLCInfo && !Globals::globalConfig.gamehasdlc.empty()) - { - if (Globals::globalConfig.gamehasdlc.isBlacklisted(game.name)) - bDownloadDLCInfo = true; - } - - // Check game specific config - if (!Globals::globalConfig.bUpdateCache) // Disable game specific config files for cache update - { - gameSpecificConfig conf; - conf.dlConf.bIgnoreDLCCount = bDownloadDLCInfo; - Util::getGameSpecificConfig(game.name, &conf); - bDownloadDLCInfo = conf.dlConf.bIgnoreDLCCount; - } - - if (bDownloadDLCInfo && !Globals::globalConfig.sGameRegex.empty()) - { - // don't download unnecessary info if user is only interested in a subset of his account - boost::regex expression(Globals::globalConfig.sGameRegex); - boost::match_results what; - if (!boost::regex_search(game.name, what, expression)) - { - bDownloadDLCInfo = false; - } - } - - if (bDownloadDLCInfo) - { - game.gamedetailsjson = this->getGameDetailsJSON(game.id); - if (!game.gamedetailsjson.empty()) - game.dlcnames = Util::getDLCNamesFromJSON(game.gamedetailsjson["dlcs"]); - } - } - games.push_back(game); } } - i++; - } while (!bAllPagesParsed); + + unsigned int platform = 0; + if (product["worksOn"]["Windows"].asBool()) + platform |= GlobalConstants::PLATFORM_WINDOWS; + if (product["worksOn"]["Mac"].asBool()) + platform |= GlobalConstants::PLATFORM_MAC; + if (product["worksOn"]["Linux"].asBool()) + platform |= GlobalConstants::PLATFORM_LINUX; + + // Skip if not new and flag is set + if (Globals::globalConfig.bNew && !game.isnew) + continue; + + // Skip if platform doesn't match + if (Globals::globalConfig.bPlatformDetection && !(platform & Globals::globalConfig.dlConf.iInstallerPlatform)) + continue; + + // Filter the game list + if (!Globals::globalConfig.sGameRegex.empty()) + { + boost::regex expression(Globals::globalConfig.sGameRegex); + boost::match_results what; + if (!boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex + continue; + } + + if (Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC) + { + int dlcCount = product["dlcCount"].asInt(); + + bool bDownloadDLCInfo = (dlcCount != 0); + + if (!bDownloadDLCInfo && !Globals::globalConfig.sIgnoreDLCCountRegex.empty()) + { + boost::regex expression(Globals::globalConfig.sIgnoreDLCCountRegex); + boost::match_results what; + if (boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex + { + bDownloadDLCInfo = true; + } + } + + // Check game specific config + if (!Globals::globalConfig.bUpdateCache) // Disable game specific config files for cache update + { + gameSpecificConfig conf; + conf.dlConf.bIgnoreDLCCount = bDownloadDLCInfo; + Util::getGameSpecificConfig(game.name, &conf); + bDownloadDLCInfo = conf.dlConf.bIgnoreDLCCount; + } + + if (bDownloadDLCInfo && !Globals::globalConfig.sGameRegex.empty()) + { + // don't download unnecessary info if user is only interested in a subset of his account + boost::regex expression(Globals::globalConfig.sGameRegex); + boost::match_results what; + if (!boost::regex_search(game.name, what, expression)) + { + bDownloadDLCInfo = false; + } + } + + if (bDownloadDLCInfo) + { + game.gamedetailsjson = this->getGameDetailsJSON(game.id); + if (!game.gamedetailsjson.empty()) + game.dlcnames = Util::getDLCNamesFromJSON(game.gamedetailsjson["dlcs"]); + } + } + games.push_back(game); + } std::cerr << std::endl; if (Globals::globalConfig.bIncludeHiddenProducts)