This commit is contained in:
Sude 2014-07-02 19:57:53 +03:00
commit 4e55b0677f
9 changed files with 322 additions and 25 deletions

View File

@ -45,9 +45,9 @@ OBJDIR_RELEASE = obj/Release
DEP_RELEASE =
OUT_RELEASE = bin/Release/lgogdownloader
OBJ_DEBUG = $(OBJDIR_DEBUG)/main.o $(OBJDIR_DEBUG)/src/api.o $(OBJDIR_DEBUG)/src/downloader.o $(OBJDIR_DEBUG)/src/progressbar.o $(OBJDIR_DEBUG)/src/util.o
OBJ_DEBUG = $(OBJDIR_DEBUG)/main.o $(OBJDIR_DEBUG)/src/api.o $(OBJDIR_DEBUG)/src/downloader.o $(OBJDIR_DEBUG)/src/progressbar.o $(OBJDIR_DEBUG)/src/util.o $(OBJDIR_DEBUG)/src/blacklist.o
OBJ_RELEASE = $(OBJDIR_RELEASE)/main.o $(OBJDIR_RELEASE)/src/api.o $(OBJDIR_RELEASE)/src/downloader.o $(OBJDIR_RELEASE)/src/progressbar.o $(OBJDIR_RELEASE)/src/util.o
OBJ_RELEASE = $(OBJDIR_RELEASE)/main.o $(OBJDIR_RELEASE)/src/api.o $(OBJDIR_RELEASE)/src/downloader.o $(OBJDIR_RELEASE)/src/progressbar.o $(OBJDIR_RELEASE)/src/util.o $(OBJDIR_RELEASE)/src/blacklist.o
all: debug release
@ -80,6 +80,9 @@ $(OBJDIR_DEBUG)/src/progressbar.o: src/progressbar.cpp
$(OBJDIR_DEBUG)/src/util.o: src/util.cpp
$(CXX) $(CFLAGS_DEBUG) $(INC_DEBUG) -c src/util.cpp -o $(OBJDIR_DEBUG)/src/util.o
$(OBJDIR_DEBUG)/src/blacklist.o: src/blacklist.cpp
$(CXX) $(CFLAGS_DEBUG) $(INC_DEBUG) -c src/blacklist.cpp -o $(OBJDIR_DEBUG)/src/blacklist.o
clean_debug:
rm -f $(OBJ_DEBUG) $(OUT_DEBUG)
rm -rf bin/Debug
@ -117,6 +120,9 @@ $(OBJDIR_RELEASE)/src/progressbar.o: src/progressbar.cpp
$(OBJDIR_RELEASE)/src/util.o: src/util.cpp
$(CXX) $(CFLAGS_RELEASE) $(INC_RELEASE) -c src/util.cpp -o $(OBJDIR_RELEASE)/src/util.o
$(OBJDIR_RELEASE)/src/blacklist.o: src/blacklist.cpp
$(CXX) $(CFLAGS_RELEASE) $(INC_RELEASE) -c src/blacklist.cpp -o $(OBJDIR_RELEASE)/src/blacklist.o
clean_release:
rm -f $(OBJ_RELEASE) $(OUT_RELEASE)
rm -rf bin/Release

38
include/blacklist.h Normal file
View File

@ -0,0 +1,38 @@
/* This program is free software. It comes without any warranty, to
* the extent permitted by applicable law. You can redistribute it
* and/or modify it under the terms of the Do What The Fuck You Want
* To Public License, Version 2, as published by Sam Hocevar. See
* http://www.wtfpl.net/ for more details. */
#ifndef BLACKLIST_H__
#define BLACKLIST_H__
#include <boost/regex.hpp>
#include <string>
#include <vector>
class Config;
class gameFile;
class BlacklistItem {
public:
unsigned int linenr; // where the blacklist item is defined in blacklist.txt
unsigned int flags;
std::string source; // source representation of the item
boost::regex regex;
};
class Blacklist
{
public:
Blacklist() {};
void initialize(const std::vector<std::string>& lines);
bool isBlacklisted(const std::string& path);
bool isBlacklisted(const std::string& path, const std::string& gamename, std::string subdirectory = "");
private:
std::vector<BlacklistItem> blacklist_;
};
#endif // BLACKLIST_H_

View File

@ -10,6 +10,8 @@
#include <iostream>
#include <curl/curl.h>
#include "blacklist.h"
class Config
{
public:
@ -48,6 +50,7 @@ class Config
std::string sConfigDirectory;
std::string sCookiePath;
std::string sConfigFilePath;
std::string sBlacklistFilePath;
std::string sOrphanRegex;
unsigned int iInstallerType;
unsigned int iInstallerLanguage;
@ -56,6 +59,7 @@ class Config
size_t iChunkSize;
curl_off_t iDownloadRate;
long int iTimeout;
Blacklist blacklist;
};
#endif // CONFIG_H__

View File

@ -18,6 +18,7 @@
namespace Util
{
std::string makeFilepath(const std::string& directory, const std::string& path, const std::string& gamename, std::string subdirectory = "");
std::string makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory = "");
std::string getFileHash(const std::string& filename, unsigned hash_id);
std::string getChunkHash(unsigned char* chunk, size_t chunk_size, unsigned hash_id);
int createXML(std::string filepath, size_t chunk_size, std::string xml_dir = std::string());

View File

@ -32,15 +32,15 @@ int main(int argc, char *argv[])
if (xdgconfig)
{
config.sConfigDirectory = (std::string)xdgconfig + "/lgogdownloader";
config.sCookiePath = config.sConfigDirectory + "/cookies.txt";
config.sConfigFilePath = config.sConfigDirectory + "/config.cfg";
}
else
{
config.sConfigDirectory = home + "/.config/lgogdownloader";
}
config.sCookiePath = config.sConfigDirectory + "/cookies.txt";
config.sConfigFilePath = config.sConfigDirectory + "/config.cfg";
}
config.sBlacklistFilePath = config.sConfigDirectory + "/blacklist.txt";
if (xdgcache)
config.sXMLDirectory = (std::string)xdgcache + "/lgogdownloader/xml";
@ -183,6 +183,26 @@ int main(int argc, char *argv[])
ifs.close();
}
}
if (boost::filesystem::exists(config.sBlacklistFilePath))
{
std::ifstream ifs(config.sBlacklistFilePath.c_str());
if (!ifs)
{
std::cout << "Could not open blacklist file: " << config.sBlacklistFilePath << std::endl;
return 1;
}
else
{
std::string line;
std::vector<std::string> lines;
while (!ifs.eof())
{
std::getline(ifs, line);
lines.push_back(std::move(line));
}
config.blacklist.initialize(lines);
}
}
if (vm.count("help"))
{

View File

@ -17,6 +17,35 @@ These games are currently offered only for the Microsoft Windows\[rg] and Apple
/Status codes:/
.nf
[blacklist]
.fi
Allows user to specify individual files that should not be downloaded or mentioned as orphans.
.sp 1
Each line in the file specifies one blacklist expression, except for empty lines and lines starting with #.
First few characters specify blacklist item type and flags.
So far, only regular expression (perl variant) are supported, so each line must start with "Rp" characters.
After a space comes the expression itself. Expressions are matched against file path relative to what was specified as \fI--directory\fP.
\fIExample black list\fP
.br
# used to store manually downloaded mods/patches/maps/, don't mention it as orphans
.br
Rp ^[^/]*/manual/.*
.br
# included with every *divinity game, once is enough
.br
Rp beyond_divinity/extras/bd_ladymageknight\.zip
.br
Rp divinity_2_developers_cut/extras/divinity_2_ladymageknight\.zip
.sp
# extra 6GB is A LOT of space if you don't actually plan to mod your game
.br
Rp the_witcher_2/extras/the_witcher_2_redkit\.zip
.br
Rp the_witcher_2/extras/extras_pack_3_hu_pl_ru_tr_zh_\.zip
.br
Rp the_witcher_2/extras/extras_pack_2_fr_it_jp_\.zip
[files]
.fi
.TP
@ -31,5 +60,11 @@ Storage for XML files
.br
If \fB$XDG_CACHE_HOME\fP is not set, it will use \fI$HOME/.cache/lgogdownloader/xml/\fP.
.TP
\fI$XDG_CONFIG_HOME/lgogdownloader/blacklist.txt\fP
Allows user to specify individual files that should not be downloaded or mentioned as orphans.
.br
It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader.
[availability]
The latest version of this distribution is available from \fIhttps://github.com/Sude-/lgogdownloader\fP

80
src/blacklist.cpp Normal file
View File

@ -0,0 +1,80 @@
/* This program is free software. It comes without any warranty, to
* the extent permitted by applicable law. You can redistribute it
* and/or modify it under the terms of the Do What The Fuck You Want
* To Public License, Version 2, as published by Sam Hocevar. See
* http://www.wtfpl.net/ for more details. */
#include "blacklist.h"
#include "config.h"
#include "api.h"
#include "util.h"
#include <iostream>
#include <utility>
enum {
BLFLAG_RX = 1 << 0,
BLFLAG_PERL = 1 << 1
};
void Blacklist::initialize(const std::vector<std::string>& lines) {
int linenr = 1;
for (auto it = lines.begin(); it != lines.end(); ++it, ++linenr) {
BlacklistItem item;
const std::string& s = *it;
if (s.length() == 0 || s[0] == '#')
continue;
std::size_t i;
for (i = 0; i < s.length() && s[i] != '\x20'; ++i) {
switch (s[i]) {
case 'R':
item.flags |= BLFLAG_RX;
break;
case 'p':
item.flags |= BLFLAG_PERL;
break;
default:
std::cout << "unknown flag '" << s[i] << "' in blacklist line " << linenr << std::endl;
break;
}
}
++i;
if (i == s.length()) {
std::cout << "empty expression in blacklist line " << linenr << std::endl;
continue;
}
if (item.flags & BLFLAG_RX) {
boost::regex::flag_type rx_flags = boost::regex::normal;
// we only support perl-like syntax for now, which is boost default (normal). Add further flag processing
// here if that changes.
rx_flags |= boost::regex::nosubs;
item.linenr = linenr;
item.source.assign(s.substr(i).c_str());
item.regex.assign(item.source, rx_flags);
blacklist_.push_back(std::move(item));
} else {
std::cout << "unknown expression type in blacklist line " << linenr << std::endl;
}
}
}
bool Blacklist::isBlacklisted(const std::string& path) {
for (auto it = blacklist_.begin(); it != blacklist_.end(); ++it) {
const BlacklistItem& item = *it;
if (item.flags & BLFLAG_RX && boost::regex_search(path, item.regex))
return true;
}
return false;
}
bool Blacklist::isBlacklisted(const std::string& path, const std::string& gamename, std::string subdirectory)
{
std::string filepath = Util::makeRelativeFilepath(path, gamename, subdirectory);
return isBlacklisted(filepath);
}

View File

@ -415,6 +415,12 @@ void Downloader::repair()
for (unsigned int j = 0; j < games[i].installers.size(); ++j)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[j].path, games[i].gamename);
if (config.blacklist.isBlacklisted(games[i].installers[j].path, games[i].gamename))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Get XML data
std::string XML = "";
@ -452,7 +458,14 @@ void Downloader::repair()
{
for (unsigned int j = 0; j < games[i].extras.size(); ++j)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, config.bSubDirectories ? "extras" : "");
std::string subdir = config.bSubDirectories ? "extras" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].extras[j].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
std::string url = gogAPI->getExtraLink(games[i].gamename, games[i].extras[j].id);
if (gogAPI->getError())
@ -472,7 +485,14 @@ void Downloader::repair()
{
for (unsigned int j = 0; j < games[i].patches.size(); ++j)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, config.bSubDirectories ? "patches" : "");
std::string subdir = config.bSubDirectories ? "patches" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].patches[j].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
std::string url = gogAPI->getPatchLink(games[i].gamename, games[i].patches[j].id);
if (gogAPI->getError())
@ -492,7 +512,14 @@ void Downloader::repair()
{
for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, config.bSubDirectories ? "languagepacks" : "");
std::string subdir = config.bSubDirectories ? "languagepacks" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].languagepacks[j].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
std::string url = gogAPI->getLanguagePackLink(games[i].gamename, games[i].languagepacks[j].id);
if (gogAPI->getError())
@ -514,7 +541,14 @@ void Downloader::repair()
{
for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "");
std::string subdir = (config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "");
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].dlcs[j].installers[k].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Get XML data
std::string XML = "";
@ -550,7 +584,13 @@ void Downloader::repair()
{
for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "");
std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].dlcs[j].patches[k].path, games[i].gamename, subdir)) {
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
std::string url = gogAPI->getPatchLink(games[i].dlcs[j].gamename, games[i].dlcs[j].patches[k].id);
if (gogAPI->getError())
@ -568,7 +608,13 @@ void Downloader::repair()
{
for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "");
std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].dlcs[j].extras[k].path, games[i].gamename, subdir)) {
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
std::string url = gogAPI->getExtraLink(games[i].dlcs[j].gamename, games[i].dlcs[j].extras[k].id);
if (gogAPI->getError())
@ -601,6 +647,12 @@ void Downloader::download()
{
// Take path from installer path because for some games the base directory for installer/extra path is not "gamename"
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[0].path, games[i].gamename);
if (config.blacklist.isBlacklisted(games[i].installers[0].path, games[i].gamename))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Get base directory from filepath
boost::match_results<std::string::const_iterator> what;
@ -621,6 +673,12 @@ void Downloader::download()
continue;
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[j].path, games[i].gamename);
if (config.blacklist.isBlacklisted(games[i].installers[j].path, games[i].gamename))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Get link
std::string url = gogAPI->getInstallerLink(games[i].gamename, games[i].installers[j].id);
@ -659,7 +717,14 @@ void Downloader::download()
continue;
}
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, config.bSubDirectories ? "extras" : "");
std::string subdir = config.bSubDirectories ? "extras" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].extras[j].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Download
if (!url.empty())
@ -693,7 +758,14 @@ void Downloader::download()
continue;
}
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, config.bSubDirectories ? "patches" : "");
std::string subdir = config.bSubDirectories ? "patches" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].patches[j].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Download
if (!url.empty())
@ -722,7 +794,14 @@ void Downloader::download()
continue;
}
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, config.bSubDirectories ? "languagepacks" : "");
std::string subdir = config.bSubDirectories ? "languagepacks" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].languagepacks[j].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Download
if (!url.empty())
@ -746,7 +825,14 @@ void Downloader::download()
{
for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "");
std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].dlcs[j].installers[k].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Get link
std::string url = gogAPI->getInstallerLink(games[i].dlcs[j].gamename, games[i].dlcs[j].installers[k].id);
@ -775,7 +861,14 @@ void Downloader::download()
{
for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "");
std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].dlcs[j].patches[k].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Get link
std::string url = gogAPI->getPatchLink(games[i].dlcs[j].gamename, games[i].dlcs[j].patches[k].id);
@ -803,7 +896,14 @@ void Downloader::download()
{
for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k)
{
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "");
std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "";
std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, subdir);
if (config.blacklist.isBlacklisted(games[i].dlcs[j].extras[k].path, games[i].gamename, subdir))
{
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
continue;
}
// Get link
std::string url = gogAPI->getExtraLink(games[i].dlcs[j].gamename, games[i].dlcs[j].extras[k].id);
@ -1982,6 +2082,7 @@ void Downloader::checkOrphans()
std::cout << "Checking for orphaned files " << i+1 << " / " << games.size() << "\r" << std::flush;
boost::filesystem::path path (config.sDirectory + games[i].gamename);
std::vector<boost::filesystem::path> filepath_vector;
std::size_t pathlen = config.sDirectory.length();
try
{
@ -1997,11 +2098,16 @@ void Downloader::checkOrphans()
if (boost::filesystem::is_regular_file(dir_iter->status()))
{
std::string filepath = dir_iter->path().string();
if (config.blacklist.isBlacklisted(filepath.substr(pathlen))) {
if (config.bVerbose)
std::cout << "skipped blacklisted file " << filepath << std::endl;
} else {
boost::regex expression(config.sOrphanRegex); // Limit to files matching the regex
boost::match_results<std::string::const_iterator> what;
if (boost::regex_search(filepath, what, expression))
filepath_vector.push_back(dir_iter->path());
}
}
dir_iter++;
}
}

View File

@ -15,6 +15,13 @@
Use gamename as base directory if specified
*/
std::string Util::makeFilepath(const std::string& directory, const std::string& path, const std::string& gamename, std::string subdirectory)
{
return directory + makeRelativeFilepath(path, gamename, subdirectory);
}
/* Create filepath relative to download base directory specified in config.
*/
std::string Util::makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory)
{
std::string filepath;
@ -23,11 +30,11 @@ std::string Util::makeFilepath(const std::string& directory, const std::string&
if (path.at(0)=='/')
{
std::string tmp_path = path.substr(1,path.length());
filepath = directory + tmp_path;
filepath = tmp_path;
}
else
{
filepath = directory + path;
filepath = path;
}
}
else
@ -37,7 +44,7 @@ std::string Util::makeFilepath(const std::string& directory, const std::string&
{
subdirectory = "/" + subdirectory;
}
filepath = directory + gamename + subdirectory + "/" + filename;
filepath = gamename + subdirectory + "/" + filename;
}
return filepath;