implemented blacklist for --download, --repair and --check-orphans

Makefile

@@ -45,9 +45,9 @@ OBJDIR_RELEASE = obj/Release
 DEP_RELEASE = 
 OUT_RELEASE = bin/Release/lgogdownloader
 
-OBJ_DEBUG = $(OBJDIR_DEBUG)/main.o $(OBJDIR_DEBUG)/src/api.o $(OBJDIR_DEBUG)/src/downloader.o $(OBJDIR_DEBUG)/src/progressbar.o $(OBJDIR_DEBUG)/src/util.o
+OBJ_DEBUG = $(OBJDIR_DEBUG)/main.o $(OBJDIR_DEBUG)/src/api.o $(OBJDIR_DEBUG)/src/downloader.o $(OBJDIR_DEBUG)/src/progressbar.o $(OBJDIR_DEBUG)/src/util.o $(OBJDIR_DEBUG)/src/blacklist.o
 
-OBJ_RELEASE = $(OBJDIR_RELEASE)/main.o $(OBJDIR_RELEASE)/src/api.o $(OBJDIR_RELEASE)/src/downloader.o $(OBJDIR_RELEASE)/src/progressbar.o $(OBJDIR_RELEASE)/src/util.o
+OBJ_RELEASE = $(OBJDIR_RELEASE)/main.o $(OBJDIR_RELEASE)/src/api.o $(OBJDIR_RELEASE)/src/downloader.o $(OBJDIR_RELEASE)/src/progressbar.o $(OBJDIR_RELEASE)/src/util.o $(OBJDIR_RELEASE)/src/blacklist.o
 
 all: debug release
 
@@ -80,6 +80,9 @@ $(OBJDIR_DEBUG)/src/progressbar.o: src/progressbar.cpp
 $(OBJDIR_DEBUG)/src/util.o: src/util.cpp
 	$(CXX) $(CFLAGS_DEBUG) $(INC_DEBUG) -c src/util.cpp -o $(OBJDIR_DEBUG)/src/util.o
 
+$(OBJDIR_DEBUG)/src/blacklist.o: src/blacklist.cpp
+	$(CXX) $(CFLAGS_DEBUG) $(INC_DEBUG) -c src/blacklist.cpp -o $(OBJDIR_DEBUG)/src/blacklist.o
+
 clean_debug: 
 	rm -f $(OBJ_DEBUG) $(OUT_DEBUG)
 	rm -rf bin/Debug
@@ -117,6 +120,9 @@ $(OBJDIR_RELEASE)/src/progressbar.o: src/progressbar.cpp
 $(OBJDIR_RELEASE)/src/util.o: src/util.cpp
 	$(CXX) $(CFLAGS_RELEASE) $(INC_RELEASE) -c src/util.cpp -o $(OBJDIR_RELEASE)/src/util.o
 
+$(OBJDIR_RELEASE)/src/blacklist.o: src/blacklist.cpp
+	$(CXX) $(CFLAGS_RELEASE) $(INC_RELEASE) -c src/blacklist.cpp -o $(OBJDIR_RELEASE)/src/blacklist.o
+
 clean_release: 
 	rm -f $(OBJ_RELEASE) $(OUT_RELEASE)
 	rm -rf bin/Release

include/blacklist.h

@@ -0,0 +1,38 @@
+/* This program is free software. It comes without any warranty, to
+ * the extent permitted by applicable law. You can redistribute it
+ * and/or modify it under the terms of the Do What The Fuck You Want
+ * To Public License, Version 2, as published by Sam Hocevar. See
+ * http://www.wtfpl.net/ for more details. */
+
+#ifndef BLACKLIST_H__
+#define BLACKLIST_H__
+
+#include <boost/regex.hpp>
+#include <string>
+#include <vector>
+
+class Config;
+class gameFile;
+
+class BlacklistItem {
+    public:
+        unsigned int linenr; // where the blacklist item is defined in blacklist.txt
+        unsigned int flags;
+        std::string source; // source representation of the item
+        boost::regex regex;
+};
+
+class Blacklist
+{
+    public:
+        Blacklist() {};
+
+        void initialize(const std::vector<std::string>& lines);
+        bool isBlacklisted(const std::string& path);
+        bool isBlacklisted(const std::string& path, const std::string& gamename, std::string subdirectory = "");
+
+    private:
+        std::vector<BlacklistItem> blacklist_;
+};
+
+#endif // BLACKLIST_H_

include/config.h

@@ -10,6 +10,8 @@
 #include <iostream>
 #include <curl/curl.h>
 
+#include "blacklist.h"
+
 class Config
 {
     public:
@@ -48,6 +50,7 @@ class Config
         std::string sConfigDirectory;
         std::string sCookiePath;
         std::string sConfigFilePath;
+        std::string sBlacklistFilePath;
         std::string sOrphanRegex;
         unsigned int iInstallerType;
         unsigned int iInstallerLanguage;
@@ -56,6 +59,7 @@ class Config
         size_t iChunkSize;
         curl_off_t iDownloadRate;
         long int iTimeout;
+        Blacklist blacklist;
 };
 
 #endif // CONFIG_H__

include/util.h

@@ -18,6 +18,7 @@
 namespace Util
 {
     std::string makeFilepath(const std::string& directory, const std::string& path, const std::string& gamename, std::string subdirectory = "");
+    std::string makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory = "");
     std::string getFileHash(const std::string& filename, unsigned hash_id);
     std::string getChunkHash(unsigned char* chunk, size_t chunk_size, unsigned hash_id);
     int createXML(std::string filepath, size_t chunk_size, std::string xml_dir = std::string());

main.cpp

@@ -40,6 +40,8 @@ int main(int argc, char *argv[])
 
     config.sCookiePath = config.sConfigDirectory + "/cookies.txt";
     config.sConfigFilePath = config.sConfigDirectory + "/config.cfg";
+    config.sBlacklistFilePath = config.sConfigDirectory + "/blacklist.txt";
+
     if (xdgcache)
         config.sXMLDirectory = (std::string)xdgcache + "/lgogdownloader/xml";
     else
@@ -181,6 +183,26 @@ int main(int argc, char *argv[])
                 ifs.close();
             }
         }
+        if (boost::filesystem::exists(config.sBlacklistFilePath))
+        {
+            std::ifstream ifs(config.sBlacklistFilePath.c_str());
+            if (!ifs)
+            {
+                std::cout << "Could not open blacklist file: " << config.sBlacklistFilePath << std::endl;
+                return 1;
+            }
+            else
+            {
+                std::string line;
+                std::vector<std::string> lines;
+                while (!ifs.eof())
+                {
+                    std::getline(ifs, line);
+                    lines.push_back(std::move(line));
+                }
+                config.blacklist.initialize(lines);
+            }
+        }
 
         if (vm.count("help"))
         {

src/blacklist.cpp

@@ -0,0 +1,80 @@
+/* This program is free software. It comes without any warranty, to
+ * the extent permitted by applicable law. You can redistribute it
+ * and/or modify it under the terms of the Do What The Fuck You Want
+ * To Public License, Version 2, as published by Sam Hocevar. See
+ * http://www.wtfpl.net/ for more details. */
+
+#include "blacklist.h"
+#include "config.h"
+#include "api.h"
+#include "util.h"
+
+#include <iostream>
+#include <utility>
+
+enum {
+    BLFLAG_RX = 1 << 0,
+    BLFLAG_PERL = 1 << 1
+};
+
+void Blacklist::initialize(const std::vector<std::string>& lines) {
+    int linenr = 1;
+    for (auto it = lines.begin(); it != lines.end(); ++it, ++linenr) {
+        BlacklistItem item;
+        const std::string& s = *it;
+
+        if (s.length() == 0 || s[0] == '#')
+          continue;
+
+        std::size_t i;
+        for (i = 0; i < s.length() && s[i] != '\x20'; ++i) {
+            switch (s[i]) {
+                case 'R':
+                    item.flags |= BLFLAG_RX;
+                    break;
+                case 'p':
+                    item.flags |= BLFLAG_PERL;
+                    break;
+                default:
+                    std::cout << "unknown flag '" << s[i] << "' in blacklist line " << linenr << std::endl;
+                    break;
+            }
+        }
+        ++i;
+        if (i == s.length()) {
+            std::cout << "empty expression in blacklist line " << linenr << std::endl;
+            continue;
+        }
+        if (item.flags & BLFLAG_RX) {
+            boost::regex::flag_type rx_flags = boost::regex::normal;
+
+            // we only support perl-like syntax for now, which is boost default (normal). Add further flag processing
+            // here if that changes.
+
+            rx_flags |= boost::regex::nosubs;
+
+            item.linenr = linenr;
+            item.source.assign(s.substr(i).c_str());
+            item.regex.assign(item.source, rx_flags);
+            blacklist_.push_back(std::move(item));
+        } else {
+            std::cout << "unknown expression type in blacklist line " << linenr << std::endl;
+        }
+    }
+}
+
+bool Blacklist::isBlacklisted(const std::string& path) {
+    for (auto it = blacklist_.begin(); it != blacklist_.end(); ++it) {
+        const BlacklistItem& item = *it;
+        if (item.flags & BLFLAG_RX && boost::regex_search(path, item.regex))
+            return true;
+    }
+    return false;
+}
+
+bool Blacklist::isBlacklisted(const std::string& path, const std::string& gamename, std::string subdirectory)
+{
+    std::string filepath = Util::makeRelativeFilepath(path, gamename, subdirectory);
+    return isBlacklisted(filepath);
+}
+

src/downloader.cpp

@@ -415,6 +415,12 @@ void Downloader::repair()
             for (unsigned int j = 0; j < games[i].installers.size(); ++j)
             {
                 std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[j].path, games[i].gamename);
+                if (config.blacklist.isBlacklisted(games[i].installers[j].path, games[i].gamename))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 // Get XML data
                 std::string XML = "";
@@ -452,7 +458,14 @@ void Downloader::repair()
         {
             for (unsigned int j = 0; j < games[i].extras.size(); ++j)
             {
-                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, config.bSubDirectories ? "extras" : "");
+                std::string subdir = config.bSubDirectories ? "extras" : "";
+                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, subdir);
+                if (config.blacklist.isBlacklisted(games[i].extras[j].path, games[i].gamename, subdir))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 std::string url = gogAPI->getExtraLink(games[i].gamename, games[i].extras[j].id);
                 if (gogAPI->getError())
@@ -472,7 +485,14 @@ void Downloader::repair()
         {
             for (unsigned int j = 0; j < games[i].patches.size(); ++j)
             {
-                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, config.bSubDirectories ? "patches" : "");
+                std::string subdir = config.bSubDirectories ? "patches" : "";
+                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, subdir);
+                if (config.blacklist.isBlacklisted(games[i].patches[j].path, games[i].gamename, subdir))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 std::string url = gogAPI->getPatchLink(games[i].gamename, games[i].patches[j].id);
                 if (gogAPI->getError())
@@ -492,7 +512,14 @@ void Downloader::repair()
         {
             for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
             {
-                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, config.bSubDirectories ? "languagepacks" : "");
+                std::string subdir = config.bSubDirectories ? "languagepacks" : "";
+                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, subdir);
+                if (config.blacklist.isBlacklisted(games[i].languagepacks[j].path, games[i].gamename, subdir))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 std::string url = gogAPI->getLanguagePackLink(games[i].gamename, games[i].languagepacks[j].id);
                 if (gogAPI->getError())
@@ -514,7 +541,14 @@ void Downloader::repair()
                 {
                     for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k)
                     {
-                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "");
+                        std::string subdir = (config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "");
+                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, subdir);
+                        if (config.blacklist.isBlacklisted(games[i].dlcs[j].installers[k].path, games[i].gamename, subdir))
+                        {
+                            if (config.bVerbose)
+                                std::cout << "skipped blacklisted file " << filepath << std::endl;
+                            continue;
+                        }
 
                         // Get XML data
                         std::string XML = "";
@@ -550,7 +584,13 @@ void Downloader::repair()
                 {
                     for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k)
                     {
-                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "");
+                        std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "";
+                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, subdir);
+                        if (config.blacklist.isBlacklisted(games[i].dlcs[j].patches[k].path, games[i].gamename, subdir)) {
+                            if (config.bVerbose)
+                                std::cout << "skipped blacklisted file " << filepath << std::endl;
+                            continue;
+                        }
 
                         std::string url = gogAPI->getPatchLink(games[i].dlcs[j].gamename, games[i].dlcs[j].patches[k].id);
                         if (gogAPI->getError())
@@ -568,7 +608,13 @@ void Downloader::repair()
                 {
                     for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k)
                     {
-                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "");
+                        std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "";
+                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, subdir);
+                        if (config.blacklist.isBlacklisted(games[i].dlcs[j].extras[k].path, games[i].gamename, subdir)) {
+                            if (config.bVerbose)
+                                std::cout << "skipped blacklisted file " << filepath << std::endl;
+                            continue;
+                        }
 
                         std::string url = gogAPI->getExtraLink(games[i].dlcs[j].gamename, games[i].dlcs[j].extras[k].id);
                         if (gogAPI->getError())
@@ -601,6 +647,12 @@ void Downloader::download()
             {
                 // Take path from installer path because for some games the base directory for installer/extra path is not "gamename"
                 std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[0].path, games[i].gamename);
+                if (config.blacklist.isBlacklisted(games[i].installers[0].path, games[i].gamename))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 // Get base directory from filepath
                 boost::match_results<std::string::const_iterator> what;
@@ -621,6 +673,12 @@ void Downloader::download()
                     continue;
 
                 std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[j].path, games[i].gamename);
+                if (config.blacklist.isBlacklisted(games[i].installers[j].path, games[i].gamename))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 // Get link
                 std::string url = gogAPI->getInstallerLink(games[i].gamename, games[i].installers[j].id);
@@ -659,7 +717,14 @@ void Downloader::download()
                     continue;
                 }
 
-                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, config.bSubDirectories ? "extras" : "");
+                std::string subdir = config.bSubDirectories ? "extras" : "";
+                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, subdir);
+                if (config.blacklist.isBlacklisted(games[i].extras[j].path, games[i].gamename, subdir))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 // Download
                 if (!url.empty())
@@ -693,7 +758,14 @@ void Downloader::download()
                     continue;
                 }
 
-                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, config.bSubDirectories ? "patches" : "");
+                std::string subdir = config.bSubDirectories ? "patches" : "";
+                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, subdir);
+                if (config.blacklist.isBlacklisted(games[i].patches[j].path, games[i].gamename, subdir))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 // Download
                 if (!url.empty())
@@ -722,7 +794,14 @@ void Downloader::download()
                     continue;
                 }
 
-                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, config.bSubDirectories ? "languagepacks" : "");
+                std::string subdir = config.bSubDirectories ? "languagepacks" : "";
+                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, subdir);
+                if (config.blacklist.isBlacklisted(games[i].languagepacks[j].path, games[i].gamename, subdir))
+                {
+                    if (config.bVerbose)
+                        std::cout << "skipped blacklisted file " << filepath << std::endl;
+                    continue;
+                }
 
                 // Download
                 if (!url.empty())
@@ -746,7 +825,14 @@ void Downloader::download()
                 {
                     for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k)
                     {
-                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "");
+                        std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename : "";
+                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].installers[k].path, games[i].gamename, subdir);
+                        if (config.blacklist.isBlacklisted(games[i].dlcs[j].installers[k].path, games[i].gamename, subdir))
+                        {
+                            if (config.bVerbose)
+                                std::cout << "skipped blacklisted file " << filepath << std::endl;
+                            continue;
+                        }
 
                         // Get link
                         std::string url = gogAPI->getInstallerLink(games[i].dlcs[j].gamename, games[i].dlcs[j].installers[k].id);
@@ -775,7 +861,14 @@ void Downloader::download()
                 {
                     for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k)
                     {
-                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "");
+                        std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/patches" : "";
+                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].patches[k].path, games[i].gamename, subdir);
+                        if (config.blacklist.isBlacklisted(games[i].dlcs[j].patches[k].path, games[i].gamename, subdir))
+                        {
+                            if (config.bVerbose)
+                                std::cout << "skipped blacklisted file " << filepath << std::endl;
+                            continue;
+                        }
 
                         // Get link
                         std::string url = gogAPI->getPatchLink(games[i].dlcs[j].gamename, games[i].dlcs[j].patches[k].id);
@@ -803,7 +896,14 @@ void Downloader::download()
                 {
                     for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k)
                     {
-                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "");
+                        std::string subdir = config.bSubDirectories ? "dlc/" + games[i].dlcs[j].gamename + "/extras" : "";
+                        std::string filepath = Util::makeFilepath(config.sDirectory, games[i].dlcs[j].extras[k].path, games[i].gamename, subdir);
+                        if (config.blacklist.isBlacklisted(games[i].dlcs[j].extras[k].path, games[i].gamename, subdir))
+                        {
+                            if (config.bVerbose)
+                                std::cout << "skipped blacklisted file " << filepath << std::endl;
+                            continue;
+                        }
 
                         // Get link
                         std::string url = gogAPI->getExtraLink(games[i].dlcs[j].gamename, games[i].dlcs[j].extras[k].id);
@@ -1982,6 +2082,7 @@ void Downloader::checkOrphans()
         std::cout << "Checking for orphaned files " << i+1 << " / " << games.size() << "\r" << std::flush;
         boost::filesystem::path path (config.sDirectory + games[i].gamename);
         std::vector<boost::filesystem::path> filepath_vector;
+        std::size_t pathlen = config.sDirectory.length();
 
         try
         {
@@ -1997,11 +2098,16 @@ void Downloader::checkOrphans()
                         if (boost::filesystem::is_regular_file(dir_iter->status()))
                         {
                             std::string filepath = dir_iter->path().string();
+                            if (config.blacklist.isBlacklisted(filepath.substr(pathlen))) {
+                                if (config.bVerbose)
+                                    std::cout << "skipped blacklisted file " << filepath << std::endl;
+                            } else {
                                 boost::regex expression(config.sOrphanRegex); // Limit to files matching the regex
                                 boost::match_results<std::string::const_iterator> what;
                                 if (boost::regex_search(filepath, what, expression))
                                     filepath_vector.push_back(dir_iter->path());
                             }
+                        }
                         dir_iter++;
                     }
                 }

src/util.cpp

@@ -15,6 +15,13 @@
     Use gamename as base directory if specified
 */
 std::string Util::makeFilepath(const std::string& directory, const std::string& path, const std::string& gamename, std::string subdirectory)
+{
+    return directory + makeRelativeFilepath(path, gamename, subdirectory);
+}
+
+/* Create filepath relative to download base directory specified in config.
+ */
+std::string Util::makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory)
 {
     std::string filepath;
 
@@ -23,11 +30,11 @@ std::string Util::makeFilepath(const std::string& directory, const std::string&
         if (path.at(0)=='/')
         {
             std::string tmp_path = path.substr(1,path.length());
-            filepath = directory + tmp_path;
+            filepath = tmp_path;
         }
         else
         {
-            filepath = directory + path;
+            filepath = path;
         }
     }
     else
@@ -37,7 +44,7 @@ std::string Util::makeFilepath(const std::string& directory, const std::string&
         {
             subdirectory = "/" + subdirectory;
         }
-        filepath = directory + gamename + subdirectory + "/" + filename;
+        filepath = gamename + subdirectory + "/" + filename;
     }
 
     return filepath;