Mirrors/lgogdownloader
2
0
Fork 0
mirror of https://github.com/Sude-/lgogdownloader.git synced 2024-11-20 11:49:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add support for caching game details

Browse Source 
Helps with large libraries when running the downloader multiple times.
Getting game details for many games takes a long time. Caching the game details makes the process much faster for subsequent runs.

Game details are cached to "$XDG_CACHE_HOME/lgogdownloader/gamedetails.json"
--update-cache creates and updates the cache.
--use-cache enables loading game details from cache.
--cache-valid specifies how long cached game details are considered valid
This commit is contained in:
Sude 2014-10-16 11:05:57 +03:00
parent a6da2e5bea
commit 9235ee8b4a
8 changed files with 306 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/config.h
View File

@ -40,8 +40,11 @@ class Config
bool bResetConfig;
bool bReport;
bool bSubDirectories;
bool bUseCache;
bool bUpdateCache;
std::string sGameRegex;
std::string sDirectory;
std::string sCacheDirectory;
std::string sXMLFile;
std::string sXMLDirectory;
std::string sToken;
@ -64,6 +67,7 @@ class Config
unsigned int iInstallerLanguage;
int iRetries;
int iWait;
int iCacheValid;
size_t iChunkSize;
curl_off_t iDownloadRate;
long int iTimeout;

6
include/downloader.h
View File

@ -25,6 +25,7 @@
#include "api.h"
#include "progressbar.h"
#include <curl/curl.h>
#include <jsoncpp/json/json.h>
#include <ctime>
#include <fstream>
@ -65,6 +66,7 @@ class Downloader
void download();
void checkOrphans();
void checkStatus();
void updateCache();
CURL* curlhandle;
Timer timer;
Config config;
@ -81,7 +83,9 @@ class Downloader
std::string getResponse(const std::string& url);
std::string getLocalFileHash(const std::string& filepath, const std::string& gamename = std::string());
std::string getRemoteFileHash(const std::string& gamename, const std::string& id);
int loadGameDetailsCache();
int saveGameDetailsCache();
std::vector<gameDetails> getGameDetailsFromJsonNode(Json::Value root, const int& recursion_level = 0);
int HTTP_Login(const std::string& email, const std::string& password);
std::vector<gameItem> getGames();
std::vector<gameItem> getFreeGames();

2
include/gamedetails.h
View File

@ -7,6 +7,7 @@
#include <iostream>
#include <vector>
#include <jsoncpp/json/json.h>
class gameDetails
{
@ -21,6 +22,7 @@ class gameDetails
std::string title;
std::string icon;;
void makeFilepaths(const Config& config);
Json::Value getDetailsAsJson();
virtual ~gameDetails();
protected:
private:

3
include/gamefile.h
View File

@ -5,10 +5,12 @@
#include <iostream>
#include <vector>
#include <jsoncpp/json/json.h>
class gameFile
{
public:
gameFile();
gameFile(const int& t_updated, const std::string& t_id, const std::string& t_name, const std::string& t_path, const std::string& t_size, const unsigned int& t_language = GlobalConstants::LANGUAGE_EN, const unsigned int& t_platform = GlobalConstants::PLATFORM_WINDOWS, const int& t_silent = 0);
int updated;
std::string id;
@ -20,6 +22,7 @@ class gameFile
int silent;
void setFilepath(const std::string& path);
std::string getFilepath();
Json::Value getAsJson();
virtual ~gameFile();
protected:
private:

17
main.cpp
View File

@ -34,6 +34,13 @@ int main(int argc, char *argv[])
char *xdgcache = getenv("XDG_CACHE_HOME");
std::string home = (std::string)getenv("HOME");
if (xdgcache)
config.sCacheDirectory = (std::string)xdgcache + "/lgogdownloader";
else
config.sCacheDirectory = home + "/.cache/lgogdownloader";
config.sXMLDirectory = config.sCacheDirectory + "/xml";
// Create help text for --platform option
std::string platform_text = "Select which installers are downloaded\n";
unsigned int platform_sum = 0;
@ -104,6 +111,7 @@ int main(int argc, char *argv[])
("reset-config", bpo::value<bool>(&config.bResetConfig)->zero_tokens()->default_value(false), "Reset config settings to default")
("report", bpo::value<std::string>(&config.sReportFilePath)->implicit_value("lgogdownloader-report.log"), "Save report of downloaded/repaired files to specified file\nDefault filename: lgogdownloader-report.log")
("no-cover", bpo::value<bool>(&bNoCover)->zero_tokens()->default_value(false), "Don't download cover images. Overrides --cover option.\nUseful for making exceptions when \"cover\" is set to true in config file.")
("update-cache", bpo::value<bool>(&config.bUpdateCache)->zero_tokens()->default_value(false), "Update game details cache")
;
// Commandline options (config file)
options_cli_cfg.add_options()
@ -138,6 +146,8 @@ int main(int argc, char *argv[])
("subdir-language-packs", bpo::value<std::string>(&config.sLanguagePackSubdir)->default_value("languagepacks"), ("Set subdirectory for language packs" + subdir_help_text).c_str())
("subdir-dlc", bpo::value<std::string>(&config.sDLCSubdir)->default_value("dlc/%dlcname%"), ("Set subdirectory for dlc" + subdir_help_text).c_str())
("subdir-game", bpo::value<std::string>(&config.sGameSubdir)->default_value("%gamename%"), ("Set subdirectory for game" + subdir_help_text).c_str())
("use-cache", bpo::value<bool>(&config.bUseCache)->zero_tokens()->default_value(false), ("Use game details cache"))
("cache-valid", bpo::value<int>(&config.iCacheValid)->default_value(2880), ("Set how long cached game details are valid (in minutes)\nDefault: 2880 minutes (48 hours)"))
;
// Options read from config file
options_cfg_only.add_options()
@ -177,11 +187,6 @@ int main(int argc, char *argv[])
config.sConfigFilePath = config.sConfigDirectory + "/config.cfg";
config.sBlacklistFilePath = config.sConfigDirectory + "/blacklist.txt";
if (xdgcache)
config.sXMLDirectory = (std::string)xdgcache + "/lgogdownloader/xml";
else
config.sXMLDirectory = home + "/.cache/lgogdownloader/xml";
// Create lgogdownloader directories
boost::filesystem::path path = config.sXMLDirectory;
if (!boost::filesystem::exists(path))
@ -438,6 +443,8 @@ int main(int argc, char *argv[])
return 1;
}
}
else if (config.bUpdateCache)
downloader.updateCache();
else if (config.bUpdateCheck) // Update check has priority over download and list
downloader.updateCheck();
else if (config.bRepair) // Repair file

233
src/downloader.cpp
View File

@ -205,6 +205,31 @@ void Downloader::getGameList()
*/
int Downloader::getGameDetails()
{
if (config.bUseCache)
{
int result = this->loadGameDetailsCache();
if (result == 0)
{
for (unsigned int i = 0; i < this->games.size(); ++i)
this->games[i].makeFilepaths(config);
return 0;
}
else
{
if (result == 1)
{
std::cout << "Cache doesn't exist." << std::endl;
std::cout << "Create cache with --update-cache" << std::endl;
}
else if (result == 3)
{
std::cout << "Cache is too old." << std::endl;
std::cout << "Update cache with --update-cache or use bigger --cache-valid" << std::endl;
}
return 1;
}
}
gameDetails game;
int updated = 0;
for (unsigned int i = 0; i < gameItems.size(); ++i)
@ -216,8 +241,11 @@ int Downloader::getGameDetails()
conf.bDLC = config.bDLC;
conf.iInstallerLanguage = config.iInstallerLanguage;
conf.iInstallerType = config.iInstallerType;
if (Util::getGameSpecificConfig(gameItems[i].name, &conf) > 0)
std::cout << std::endl << gameItems[i].name << " - Language: " << conf.iInstallerLanguage << ", Platform: " << conf.iInstallerType << ", DLC: " << (conf.bDLC ? "true" : "false") << std::endl;
if (!config.bUpdateCache) // Disable game specific config files for cache update
{
if (Util::getGameSpecificConfig(gameItems[i].name, &conf) > 0)
std::cout << std::endl << gameItems[i].name << " - Language: " << conf.iInstallerLanguage << ", Platform: " << conf.iInstallerType << ", DLC: " << (conf.bDLC ? "true" : "false") << std::endl;
}
game = gogAPI->getGameDetails(gameItems[i].name, conf.iInstallerType, conf.iInstallerLanguage, config.bDuplicateHandler);
if (!gogAPI->getError())
@ -2617,3 +2645,204 @@ std::string Downloader::getRemoteFileHash(const std::string& gamename, const std
}
return remoteHash;
}
/* Load game details from cache file
returns 0 if successful
returns 1 if cache file doesn't exist
returns 2 if JSON parsing failed
returns 3 if cache is too old
returns 4 if JSON doesn't contain "games" node
*/
int Downloader::loadGameDetailsCache()
{
int res = 0;
std::string cachepath = config.sCacheDirectory + "/gamedetails.json";
// Make sure file exists
boost::filesystem::path path = cachepath;
if (!boost::filesystem::exists(path)) {
return res = 1;
}
bptime::ptime now = bptime::second_clock::local_time();
bptime::ptime cachedate;
std::ifstream json(cachepath, std::ifstream::binary);
Json::Value root;
Json::Reader *jsonparser = new Json::Reader;
if (jsonparser->parse(json, root))
{
if (root.isMember("date"))
{
cachedate = bptime::from_iso_string(root["date"].asString());
if ((now - cachedate) > bptime::minutes(config.iCacheValid))
{
// cache is too old
delete jsonparser;
json.close();
return res = 3;
}
}
if (root.isMember("games"))
{
this->games = getGameDetailsFromJsonNode(root["games"]);
res = 0;
}
else
{
res = 4;
}
}
else
{
res = 2;
std::cout << "Failed to parse cache" << std::endl;
std::cout << jsonparser->getFormatedErrorMessages() << std::endl;
}
delete jsonparser;
if (json)
json.close();
return res;
}
/* Save game details to cache file
returns 0 if successful
returns 1 if fails
*/
int Downloader::saveGameDetailsCache()
{
int res = 0;
std::string cachepath = config.sCacheDirectory + "/gamedetails.json";
Json::Value json;
json["date"] = bptime::to_iso_string(bptime::second_clock::local_time());
for (unsigned int i = 0; i < this->games.size(); ++i)
json["games"].append(this->games[i].getDetailsAsJson());
std::ofstream ofs(cachepath);
if (!ofs)
{
res = 1;
}
else
{
Json::StyledStreamWriter jsonwriter;
jsonwriter.write(ofs, json);
ofs.close();
}
return res;
}
std::vector<gameDetails> Downloader::getGameDetailsFromJsonNode(Json::Value root, const int& recursion_level)
{
std::vector<gameDetails> details;
// If root node is not array and we use root.size() it will return the number of nodes --> limit to 1 "array" node to make sure it is handled properly
for (unsigned int i = 0; i < (root.isArray() ? root.size() : 1); ++i)
{
Json::Value gameDetailsNode = (root.isArray() ? root[i] : root); // This json node can be array or non-array so take that into account
gameDetails game;
game.gamename = gameDetailsNode["gamename"].asString();
// DLCs are handled as part of the game so make sure that filtering is done with base game name
if (recursion_level == 0) // recursion level is 0 when handling base game
{
boost::regex expression(config.sGameRegex);
boost::match_results<std::string::const_iterator> what;
if (!boost::regex_search(game.gamename, what, expression)) // Check if name matches the specified regex
continue;
}
game.title = gameDetailsNode["title"].asString();
game.icon = gameDetailsNode["icon"].asString();
// Make a vector of valid node names to make things easier
std::vector<std::string> nodes;
nodes.push_back("extras");
nodes.push_back("installers");
nodes.push_back("patches");
nodes.push_back("languagepacks");
nodes.push_back("dlcs");
gameSpecificConfig conf;
conf.bDLC = config.bDLC;
conf.iInstallerLanguage = config.iInstallerLanguage;
conf.iInstallerType = config.iInstallerType;
if (Util::getGameSpecificConfig(game.gamename, &conf) > 0)
std::cout << game.gamename << " - Language: " << conf.iInstallerLanguage << ", Platform: " << conf.iInstallerType << ", DLC: " << (conf.bDLC ? "true" : "false") << std::endl;
for (unsigned int j = 0; j < nodes.size(); ++j)
{
std::string nodeName = nodes[j];
if (gameDetailsNode.isMember(nodeName))
{
Json::Value fileDetailsNodeVector = gameDetailsNode[nodeName];
for (unsigned int index = 0; index < fileDetailsNodeVector.size(); ++index)
{
Json::Value fileDetailsNode = fileDetailsNodeVector[index];
gameFile fileDetails;
if (nodeName != "dlcs")
{
fileDetails.updated = fileDetailsNode["updated"].asInt();
fileDetails.id = fileDetailsNode["id"].asString();
fileDetails.name = fileDetailsNode["name"].asString();
fileDetails.path = fileDetailsNode["path"].asString();
fileDetails.size = fileDetailsNode["size"].asString();
fileDetails.platform = fileDetailsNode["platform"].asUInt();
fileDetails.language = fileDetailsNode["language"].asUInt();
fileDetails.silent = fileDetailsNode["silent"].asInt();
if (nodeName != "extras" && !(fileDetails.platform & conf.iInstallerType))
continue;
if (nodeName != "extras" && !(fileDetails.language & conf.iInstallerLanguage))
continue;
}
if (nodeName == "extras" && config.bExtras)
game.extras.push_back(fileDetails);
else if (nodeName == "installers" && config.bInstallers)
game.installers.push_back(fileDetails);
else if (nodeName == "patches" && config.bPatches)
game.patches.push_back(fileDetails);
else if (nodeName == "languagepacks" && config.bLanguagePacks)
game.languagepacks.push_back(fileDetails);
else if (nodeName == "dlcs" && conf.bDLC)
game.dlcs = this->getGameDetailsFromJsonNode(fileDetailsNode, recursion_level + 1);
}
}
}
if (!game.extras.empty() || !game.installers.empty() || !game.patches.empty() || !game.languagepacks.empty() || !game.dlcs.empty())
details.push_back(game);
}
return details;
}
void Downloader::updateCache()
{
// Make sure that all details get cached
unsigned int all_platforms = GlobalConstants::PLATFORM_WINDOWS;
unsigned int all_languages = GlobalConstants::LANGUAGE_EN;
for (unsigned int i = 0; i < GlobalConstants::PLATFORMS.size(); ++i)
all_platforms |= GlobalConstants::PLATFORMS[i].platformId;
for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i)
all_languages |= GlobalConstants::LANGUAGES[i].languageId;
config.bExtras = true;
config.bInstallers = true;
config.bPatches = true;
config.bLanguagePacks = true;
config.bDLC = true;
config.sGameRegex = ".*";
config.iInstallerLanguage = all_languages;
config.iInstallerType = all_platforms;
this->getGameList();
this->getGameDetails();
if (this->saveGameDetailsCache())
std::cout << "Failed to save cache" << std::endl;
return;
}

28
src/gamedetails.cpp
View File

@ -69,3 +69,31 @@ void gameDetails::makeFilepaths(const Config& config)
}
}
}
Json::Value gameDetails::getDetailsAsJson()
{
Json::Value json;
json["gamename"] = this->gamename;
json["title"] = this->title;
json["icon"] = this->icon;
for (unsigned int i = 0; i < this->extras.size(); ++i)
json["extras"].append(this->extras[i].getAsJson());
for (unsigned int i = 0; i < this->installers.size(); ++i)
json["installers"].append(this->installers[i].getAsJson());
for (unsigned int i = 0; i < this->patches.size(); ++i)
json["patches"].append(this->patches[i].getAsJson());
for (unsigned int i = 0; i < this->languagepacks.size(); ++i)
json["languagepacks"].append(this->languagepacks[i].getAsJson());
if (!this->dlcs.empty())
{
for (unsigned int i = 0; i < this->dlcs.size(); ++i)
{
json["dlcs"].append(this->dlcs[i].getDetailsAsJson());
}
}
return json;
}

21
src/gamefile.cpp
View File

@ -12,6 +12,11 @@ gameFile::gameFile(const int& t_updated, const std::string& t_id, const std::str
this->silent = t_silent;
}
gameFile::gameFile()
{
//ctor
}
gameFile::~gameFile()
{
//dtor
@ -26,3 +31,19 @@ std::string gameFile::getFilepath()
{
return this->filepath;
}
Json::Value gameFile::getAsJson()
{
Json::Value json;
json["updated"] = this->updated;
json["id"] = this->id;
json["name"] = this->name;
json["path"] = this->path;
json["size"] = this->size;
json["platform"] = this->platform;
json["language"] = this->language;
json["silent"] = this->silent;
return json;
}