lgogdownloader/src/downloader.cpp

/* This program is free software. It comes without any warranty, to
 * the extent permitted by applicable law. You can redistribute it
 * and/or modify it under the terms of the Do What The Fuck You Want
 * To Public License, Version 2, as published by Sam Hocevar. See
 * http://www.wtfpl.net/ for more details. */

#include "downloader.h"
#include "util.h"
#include "globalconstants.h"

#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <iostream>
#include <sstream>
#include <unistd.h>
#include <fstream>
#include <iomanip>
#include <boost/filesystem.hpp>
#include <boost/regex.hpp>
#include <boost/date_time/posix_time/posix_time_types.hpp>
#include <tinyxml.h>
#include <jsoncpp/json/json.h>
#include <htmlcxx/html/ParserDom.h>

namespace bptime = boost::posix_time;

Downloader::Downloader(Config &conf)
{
    this->config = conf;
}

Downloader::~Downloader()
{
    if (config.bReport)
        if (this->report_ofs)
            this->report_ofs.close();
    delete progressbar;
    delete gogAPI;
    curl_easy_cleanup(curlhandle);
    curl_global_cleanup();
}


/* Initialize the downloader
    returns 0 if successful
    returns 1 if failed
*/
int Downloader::init()
{
    this->resume_position = 0;
    this->retries = 0;

    // Initialize curl and set curl options
    curl_global_init(CURL_GLOBAL_ALL);
    curlhandle = curl_easy_init();
    curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1);
    curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, config.sVersionString.c_str());
    curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0);
    curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, config.iTimeout);
    curl_easy_setopt(curlhandle, CURLOPT_PROGRESSDATA, this);
    curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true);
    curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, config.sCookiePath.c_str());
    curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, config.sCookiePath.c_str());
    curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, config.bVerifyPeer);
    curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, config.bVerbose);
    curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
    curl_easy_setopt(curlhandle, CURLOPT_READFUNCTION, Downloader::readData);
    curl_easy_setopt(curlhandle, CURLOPT_PROGRESSFUNCTION, Downloader::progressCallback);
    curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, config.iDownloadRate);

    // Create new API handle and set curl options for the API
    gogAPI = new API(config.sToken, config.sSecret);
    gogAPI->curlSetOpt(CURLOPT_VERBOSE, config.bVerbose);
    gogAPI->curlSetOpt(CURLOPT_SSL_VERIFYPEER, config.bVerifyPeer);
    gogAPI->curlSetOpt(CURLOPT_CONNECTTIMEOUT, config.iTimeout);

    progressbar = new ProgressBar(config.bUnicode, config.bColor);

    bool bInitOK = gogAPI->init(); // Initialize the API
    if (config.bLogin || !bInitOK)
        return this->login();

    if (config.bCover && config.bDownload && !config.bUpdateCheck)
        coverXML = this->getResponse("https://sites.google.com/site/gogdownloader/covers.xml");

    if (!config.bUpdateCheck) // updateCheck() calls getGameList() if needed
        this->getGameList();

    if (config.bReport && (config.bDownload || config.bRepair))
    {
        this->report_ofs.open("lgogdownloader-report.log");
        if (!this->report_ofs)
        {
            std::cout << "Failed to create lgogdownloader-report.log" << std::endl;
            return 1;
        }
    }

    return 0;
}

/* Login
    returns 1 if login fails
    returns 0 if successful
*/
int Downloader::login()
{
    char *pwd;
    std::string email;
    std::cout << "Email: ";
    std::getline(std::cin,email);
    pwd = getpass("Password: ");
    std::string password = (std::string)pwd;
    if (email.empty() || password.empty())
    {
        std::cout << "Email and/or password empty" << std::endl;
        return 1;
    }
    else
    {
        // Login to website
        if (!HTTP_Login(email, password))
        {
            std::cout << "HTTP: Login failed" << std::endl;
            return 1;
        }
        else
        {
            std::cout << "HTTP: Login successful" << std::endl;
        }
        // Login to API
        if (!gogAPI->login(email, password))
        {
            std::cout << "API: Login failed" << std::endl;
            return 1;
        }
        else
        {
            std::cout << "API: Login successful" << std::endl;
            // Save token and secret to config file
            std::ofstream ofs(config.sConfigFilePath.c_str());
            if (ofs)
            {
                ofs << "token = " << gogAPI->getToken() << std::endl << "secret = " << gogAPI->getSecret() << std::endl;
                ofs.close();
                return 0;
            }
            else
            {
                std::cout << "Failed to create config: " << config.sConfigFilePath << std::endl;
                return 1;
            }
        }
    }
}

void Downloader::updateCheck()
{
    std::cout << "New forum replies: " << gogAPI->user.notifications_forum << std::endl;
    std::cout << "New private messages: " << gogAPI->user.notifications_messages << std::endl;
    std::cout << "Updated games: " << gogAPI->user.notifications_games << std::endl;

    if (gogAPI->user.notifications_games)
    {
        config.sGameRegex = ".*"; // Always check all games
        if (config.bList || config.bListDetails || config.bDownload)
        {
            if (config.bList)
                config.bListDetails = true; // Always list details
            this->getGameList();
            if (config.bDownload)
                this->download();
            else
                this->listGames();
        }
    }
}

void Downloader::getGameList()
{
    gameNamesIds = this->getGames();

    // Filter the game list
    if (!config.sGameRegex.empty())
    {
        // GameRegex filter aliases
        if (config.sGameRegex == "all")
            config.sGameRegex = ".*";

        if (config.sGameRegex == "free")
        {
            gameNamesIds = this->getFreeGames();
        }
        else
        {   // Filter the names
            std::vector<std::pair<std::string,std::string>> gameNamesIdsFiltered;
            boost::regex expression(config.sGameRegex);
            boost::match_results<std::string::const_iterator> what;
            for (unsigned int i = 0; i < gameNamesIds.size(); ++i)
            {
                if (boost::regex_search(gameNamesIds[i].first, what, expression)) // Check if name matches the specified regex
                    gameNamesIdsFiltered.push_back(gameNamesIds[i]);
            }
            gameNamesIds = gameNamesIdsFiltered;
        }
    }
}

/* Get detailed info about the games
    returns 0 if successful
    returns 1 if fails
*/
int Downloader::getGameDetails()
{
    gameDetails game;
    int updated = 0;
    for (unsigned int i = 0; i < gameNamesIds.size(); ++i)
    {
        std::cout << "Getting game info " << i+1 << " / " << gameNamesIds.size() << "\r" << std::flush;
        game = gogAPI->getGameDetails(gameNamesIds[i].first, config.iInstallerType, config.iInstallerLanguage, config.bDuplicateHandler);
        if (!gogAPI->getError())
        {
            if (game.extras.empty() && config.bExtras) // Try to get extras from account page if API didn't return any extras
            {
                game.extras = this->getExtras(gameNamesIds[i].first, gameNamesIds[i].second);
            }
            if (!config.bUpdateCheck)
                games.push_back(game);
            else
            { // Update check, only add games that have updated files
                for (unsigned int j = 0; j < game.installers.size(); ++j)
                {
                    if (game.installers[j].updated)
                    {
                        games.push_back(game);
                        updated++;
                        break; // add the game only once
                    }
                }
                if (updated >= gogAPI->user.notifications_games)
                { // Gone through all updated games. No need to go through the rest.
                    std::cout << std::endl << "Got info for all updated games. Moving on..." << std::endl;
                    break;
                }
            }
        }
        else
        {
            std::cout << gogAPI->getErrorMessage() << std::endl;
            return 1;
        }
    }
    std::cout << std::endl;
    return 0;
}

void Downloader::listGames()
{
    if (config.bListDetails) // Detailed list
    {
        if (this->games.empty())
            this->getGameDetails();

        for (unsigned int i = 0; i < games.size(); ++i)
        {
            std::cout   << "gamename: " << games[i].gamename << std::endl
                        << "title: " << games[i].title << std::endl
                        << "icon: " << "http://static.gog.com" << games[i].icon << std::endl;
            // List installers
            if (config.bInstallers)
            {
                std::cout << "installers: " << std::endl;
                for (unsigned int j = 0; j < games[i].installers.size(); ++j)
                {
                    if (!config.bUpdateCheck || games[i].installers[j].updated) // Always list updated files
                    {
                        std::string languages;
                        for (unsigned int k = 0; k < GlobalConstants::LANGUAGES.size(); k++) // Check which languages the installer supports
                        {
                            if (games[i].installers[j].language & GlobalConstants::LANGUAGES[k].languageId)
                                languages += (languages.empty() ? "" : ", ")+GlobalConstants::LANGUAGES[k].languageString;
                        }

                        std::cout   << "\tid: " << games[i].installers[j].id << std::endl
                                    << "\tname: " << games[i].installers[j].name << std::endl
                                    << "\tpath: " << games[i].installers[j].path << std::endl
                                    << "\tsize: " << games[i].installers[j].size << std::endl
                                    << "\tupdated: " << (games[i].installers[j].updated ? "True" : "False") << std::endl
                                    << "\tlanguage: " << languages << std::endl
                                    << std::endl;
                    }
                }
            }
            // List extras
            if (config.bExtras && !config.bUpdateCheck && !games[i].extras.empty())
            {
                std::cout << "extras: " << std::endl;
                for (unsigned int j = 0; j < games[i].extras.size(); ++j)
                {
                    std::cout   << "\tid: " << games[i].extras[j].id << std::endl
                                << "\tname: " << games[i].extras[j].name << std::endl
                                << "\tpath: " << games[i].extras[j].path << std::endl
                                << "\tsize: " << games[i].extras[j].size << std::endl
                                << std::endl;
                }
            }
            // List patches
            if (config.bPatches && !config.bUpdateCheck && !games[i].patches.empty())
            {
                std::cout << "patches: " << std::endl;
                for (unsigned int j = 0; j < games[i].patches.size(); ++j)
                {
                    std::cout   << "\tid: " << games[i].patches[j].id << std::endl
                                << "\tname: " << games[i].patches[j].name << std::endl
                                << "\tpath: " << games[i].patches[j].path << std::endl
                                << "\tsize: " << games[i].patches[j].size << std::endl
                                << std::endl;
                }
            }
            // List language packs
            if (config.bLanguagePacks && !config.bUpdateCheck && !games[i].languagepacks.empty())
            {
                std::cout << "language packs: " << std::endl;
                for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
                {
                    std::cout   << "\tid: " << games[i].languagepacks[j].id << std::endl
                                << "\tname: " << games[i].languagepacks[j].name << std::endl
                                << "\tpath: " << games[i].languagepacks[j].path << std::endl
                                << "\tsize: " << games[i].languagepacks[j].size << std::endl
                                << std::endl;
                }
            }
        }
    }
    else
    {   // List game names
        for (unsigned int i = 0; i < gameNamesIds.size(); ++i)
            std::cout << gameNamesIds[i].first << std::endl;
    }

}

void Downloader::repair()
{
    if (this->games.empty())
        this->getGameDetails();

    for (unsigned int i = 0; i < games.size(); ++i)
    {
        // Installers (use remote or local file)
        if (config.bInstallers)
        {
            for (unsigned int j = 0; j < games[i].installers.size(); ++j)
            {
                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[j].path, games[i].gamename);

                // Get XML data
                std::string XML = "";
                if (config.bRemoteXML)
                {
                    XML = gogAPI->getXML(games[i].gamename, games[i].installers[j].id);
                    if (gogAPI->getError())
                    {
                        std::cout << gogAPI->getErrorMessage() << std::endl;
                        gogAPI->clearError();
                        continue;
                    }
                }

                // Repair
                bool bUseLocalXML = !config.bRemoteXML;
                if (!XML.empty() || bUseLocalXML)
                {
                    std::string url = gogAPI->getInstallerLink(games[i].gamename, games[i].installers[j].id);
                    if (gogAPI->getError())
                    {
                        std::cout << gogAPI->getErrorMessage() << std::endl;
                        gogAPI->clearError();
                        continue;
                    }
                    std::cout << "Repairing file " << filepath << std::endl;
                    this->repairFile(url, filepath, XML);
                    std::cout << std::endl;
                }
            }
        }

        // Extras (GOG doesn't provide XML data for extras, use local file)
        if (config.bExtras)
        {
            for (unsigned int j = 0; j < games[i].extras.size(); ++j)
            {
                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, config.bSubDirectories ? "extras" : "");

                std::string url = gogAPI->getExtraLink(games[i].gamename, games[i].extras[j].id);
                if (gogAPI->getError())
                {
                    std::cout << gogAPI->getErrorMessage() << std::endl;
                    gogAPI->clearError();
                    continue;
                }
                std::cout << "Repairing file " << filepath << std::endl;
                this->repairFile(url, filepath);
                std::cout << std::endl;
            }
        }

        // Patches (use remote or local file)
        if (config.bPatches)
        {
            for (unsigned int j = 0; j < games[i].patches.size(); ++j)
            {
                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, config.bSubDirectories ? "patches" : "");

                std::string url = gogAPI->getPatchLink(games[i].gamename, games[i].patches[j].id);
                if (gogAPI->getError())
                {
                    std::cout << gogAPI->getErrorMessage() << std::endl;
                    gogAPI->clearError();
                    continue;
                }
                std::cout << "Repairing file " << filepath << std::endl;
                this->repairFile(url, filepath);
                std::cout << std::endl;
            }
        }

        // Language packs (GOG doesn't provide XML data for language packs, use local file)
        if (config.bLanguagePacks)
        {
            for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
            {
                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, config.bSubDirectories ? "languagepacks" : "");

                std::string url = gogAPI->getLanguagePackLink(games[i].gamename, games[i].languagepacks[j].id);
                if (gogAPI->getError())
                {
                    std::cout << gogAPI->getErrorMessage() << std::endl;
                    gogAPI->clearError();
                    continue;
                }
                std::cout << "Repairing file " << filepath << std::endl;
                this->repairFile(url, filepath);
                std::cout << std::endl;
            }
        }

    }
}

void Downloader::download()
{
    if (this->games.empty())
        this->getGameDetails();

    for (unsigned int i = 0; i < games.size(); ++i)
    {
        // Download covers
        if (config.bCover && !config.bUpdateCheck)
        {
            if (!games[i].installers.empty())
            {
                // Take path from installer path because for some games the base directory for installer/extra path is not "gamename"
                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[0].path, games[i].gamename);

                // Get base directory from filepath
                boost::match_results<std::string::const_iterator> what;
                boost::regex expression("(.*)/.*");
                boost::regex_match(filepath, what, expression);
                std::string directory = what[1];

                this->downloadCovers(games[i].gamename, directory, coverXML);
            }
        }
        // Download installers
        if (config.bInstallers)
        {
            for (unsigned int j = 0; j < games[i].installers.size(); ++j)
            {
                // Not updated, skip to next installer
                if (config.bUpdateCheck && !games[i].installers[j].updated)
                    continue;

                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].installers[j].path, games[i].gamename);

                // Get link
                std::string url = gogAPI->getInstallerLink(games[i].gamename, games[i].installers[j].id);
                if (gogAPI->getError())
                {
                    std::cout << gogAPI->getErrorMessage() << std::endl;
                    gogAPI->clearError();
                    continue;
                }

                // Download
                if (!url.empty())
                {
                    std::string XML;
                    if (config.bRemoteXML)
                        XML = gogAPI->getXML(games[i].gamename, games[i].installers[j].id);
                    if (!games[i].installers[j].name.empty())
                        std::cout << "Dowloading: " << games[i].installers[j].name << std::endl;
                    std::cout << filepath << std::endl;
                    this->downloadFile(url, filepath, XML);
                    std::cout << std::endl;
                }
            }
        }
        // Download extras
        if (config.bExtras && !config.bUpdateCheck)
        { // Save some time and don't process extras when running update check. Extras don't have updated flag, all of them would be skipped anyway.
            for (unsigned int j = 0; j < games[i].extras.size(); ++j)
            {
                // Get link
                std::string url = gogAPI->getExtraLink(games[i].gamename, games[i].extras[j].id);
                if (gogAPI->getError())
                {
                    std::cout << gogAPI->getErrorMessage() << std::endl;
                    gogAPI->clearError();
                    continue;
                }

                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, config.bSubDirectories ? "extras" : "");

                // Download
                if (!url.empty())
                {
                    if (!games[i].extras[j].name.empty())
                        std::cout << "Dowloading: " << games[i].extras[j].name << std::endl;
                    std::cout << filepath << std::endl;
                    CURLcode result = downloadFile(url, filepath);
                    std::cout << std::endl;
                    if (result==CURLE_OK && config.sXMLFile == "automatic")
                    {
                        std::cout << "Starting automatic XML creation" << std::endl;
                        Util::createXML(filepath, config.iChunkSize, config.sXMLDirectory);
                        std::cout << std::endl;
                    }
                }
            }
        }
        // Download patches
        if (config.bPatches)
        {
            for (unsigned int j = 0; j < games[i].patches.size(); ++j)
            {
                // Get link
                std::string url = gogAPI->getPatchLink(games[i].gamename, games[i].patches[j].id);
                if (gogAPI->getError())
                {
                    std::cout << gogAPI->getErrorMessage() << std::endl;
                    gogAPI->clearError();
                    continue;
                }

                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, config.bSubDirectories ? "patches" : "");

                // Download
                if (!url.empty())
                {
                    if (!games[i].patches[j].name.empty())
                        std::cout << "Dowloading: " << games[i].gamename << " " << games[i].patches[j].name << std::endl;
                    std::cout << filepath << std::endl;
                    CURLcode result = downloadFile(url, filepath);
                    std::cout << std::endl;
                    if (result==CURLE_OK && config.sXMLFile == "automatic")
                    {
                        std::cout << "Starting automatic XML creation" << std::endl;
                        Util::createXML(filepath, config.iChunkSize, config.sXMLDirectory);
                        std::cout << std::endl;
                    }
                }
            }
        }
        // Download language packs
        if (config.bLanguagePacks)
        {
            for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
            {
                // Get link
                std::string url = gogAPI->getLanguagePackLink(games[i].gamename, games[i].languagepacks[j].id);
                if (gogAPI->getError())
                {
                    std::cout << gogAPI->getErrorMessage() << std::endl;
                    gogAPI->clearError();
                    continue;
                }

                std::string filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, config.bSubDirectories ? "languagepacks" : "");

                // Download
                if (!url.empty())
                {
                    if (!games[i].languagepacks[j].name.empty())
                        std::cout << "Dowloading: " << games[i].gamename << " " << games[i].languagepacks[j].name << std::endl;
                    std::cout << filepath << std::endl;
                    CURLcode result = downloadFile(url, filepath);
                    std::cout << std::endl;
                    if (result==CURLE_OK && config.sXMLFile == "automatic")
                    {
                        std::cout << "Starting automatic XML creation" << std::endl;
                        Util::createXML(filepath, config.iChunkSize, config.sXMLDirectory);
                        std::cout << std::endl;
                    }
                }
            }
        }
    }
}

// Download a file, resume if possible
CURLcode Downloader::downloadFile(const std::string& url, const std::string& filepath, const std::string& xml_data)
{
    CURLcode res = CURLE_RECV_ERROR; // assume network error
    bool bResume = false;
    FILE *outfile;
    size_t offset=0;

    // Get directory from filepath
    boost::filesystem::path pathname = filepath;
    std::string directory = pathname.parent_path().string();
    std::string filenameXML = pathname.filename().string() + ".xml";

    // Using local XML data for version check before resuming
    boost::filesystem::path local_xml_file;
    local_xml_file = config.sXMLDirectory + "/" + filenameXML;

    bool bSameVersion = true; // assume same version
    bool bLocalXMLExists = boost::filesystem::exists(local_xml_file); // This is additional check to see if remote xml should be saved to speed up future version checks
    std::string localHash = this->getLocalFileHash(filepath);

    if (!xml_data.empty())
    {
        // Do version check if local hash exists
        if (!localHash.empty())
        {
            TiXmlDocument remote_xml;
            remote_xml.Parse(xml_data.c_str());
            TiXmlNode *fileNodeRemote = remote_xml.FirstChild("file");
            if (fileNodeRemote)
            {
                TiXmlElement *fileElemRemote = fileNodeRemote->ToElement();
                std::string remoteHash = fileElemRemote->Attribute("md5");
                if (remoteHash != localHash)
                    bSameVersion = false;
            }
        }
    }

    // Check that directory exists and create subdirectories
    boost::filesystem::path path = directory;
    if (boost::filesystem::exists(path))
    {
        if (!boost::filesystem::is_directory(path))
        {
            std::cout << path << " is not directory" << std::endl;
            return res;
        }
    }
    else
    {
        if (!boost::filesystem::create_directories(path))
        {
            std::cout << "Failed to create directory: " << path << std::endl;
            return res;
        }
    }

    // Check if file exists
    if ((outfile=fopen(filepath.c_str(), "r"))!=NULL)
    {
        if (bSameVersion)
        {
            // File exists, resume
            if ((outfile = freopen(filepath.c_str(), "r+", outfile))!=NULL )
            {
                bResume = true;
                fseek(outfile, 0, SEEK_END);
                offset = ftell(outfile);
                curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM, offset);
                this->resume_position = offset;
            }
            else
            {
                std::cout << "Failed to reopen " << filepath << std::endl;
                return res;
            }
        }
        else
        {   // File exists but is not the same version
            fclose(outfile);
            std::cout << "Remote file is different, renaming local file" << std::endl;
            boost::filesystem::path new_name = filepath + ".old"; // Rename old file by appending ".old" to filename
            if (boost::filesystem::exists(new_name))
            {   // One even older file exists, delete the older file before renaming
                std::cout << "Old renamed file found, deleting old file" << std::endl;
                if (!boost::filesystem::remove(new_name))
                {
                    std::cout << "Failed to delete " << new_name.string() << std::endl;
                    std::cout << "Skipping file" << std::endl;
                    return res;
                }
            }
            boost::system::error_code ec;
            boost::filesystem::rename(pathname, new_name, ec); // Rename the file
            if (ec)
            {
                std::cout << "Failed to rename " << filepath << " to " << new_name.string() << std::endl;
                std::cout << "Skipping file" << std::endl;
                return res;
            }
            else
            {
                // Create new file
                if ((outfile=fopen(filepath.c_str(), "w"))!=NULL)
                {
                    curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM, 0); // start downloading from the beginning of file
                    this->resume_position = 0;
                }
                else
                {
                    std::cout << "Failed to create " << filepath << std::endl;
                    return res;
                }
            }
        }
    }
    else
    {
        // File doesn't exist, create new file
        if ((outfile=fopen(filepath.c_str(), "w"))!=NULL)
        {
            curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM, 0); // start downloading from the beginning of file
            this->resume_position = 0;
        }
        else
        {
            std::cout << "Failed to create " << filepath << std::endl;
            return res;
        }
    }

    // Save remote XML
    if (!xml_data.empty())
    {
        if ((bLocalXMLExists && (!bSameVersion || config.bRepair)) || !bLocalXMLExists)
        {
            std::ofstream ofs(local_xml_file.string().c_str());
            if (ofs)
            {
                ofs << xml_data;
                ofs.close();
            }
            else
            {
                std::cout << "Can't create " << local_xml_file.string() << std::endl;
            }
        }
    }

    curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, outfile);
    res = this->beginDownload();

    fclose(outfile);

    // Download failed and was not a resume attempt so delete the file
    if ( (res != CURLE_OK || res != CURLE_PARTIAL_FILE) && !bResume )
    {
        boost::filesystem::path path = filepath;
        if (boost::filesystem::exists(path))
            if (!boost::filesystem::remove(path))
                std::cout << "Failed to delete " << path << std::endl;
    }

    if (config.bReport)
    {
        std::string status = static_cast<std::string>(curl_easy_strerror(res));
        if (bResume && res == CURLE_RANGE_ERROR) // CURLE_RANGE_ERROR on resume attempts is not an error that user needs to know about
            status = "No error";
        std::string report_line = "Downloaded [" + status + "] " + filepath;
        this->report_ofs << report_line << std::endl;
    }

    // Retry partially downloaded file
    if (res == CURLE_PARTIAL_FILE && (this->retries < config.iRetries) )
    {
        this->retries++;
        res = this->downloadFile(url, filepath, xml_data);
    }
    else
    {
        this->retries = 0; // Reset retries counter
    }

    return res;
}

// Repair file
int Downloader::repairFile(const std::string& url, const std::string& filepath, const std::string& xml_data)
{
    int res = 0;
    FILE *outfile;
    size_t offset=0, from_offset, to_offset, filesize;
    std::string filehash;
    int chunks;
    std::vector<size_t> chunk_from, chunk_to;
    std::vector<std::string> chunk_hash;
    bool bParsingFailed = false;

    // Get filename
    boost::filesystem::path pathname = filepath;
    std::string filename = pathname.filename().string();
    std::string xml_file = config.sXMLDirectory + "/" + filename + ".xml";
    bool bFileExists = boost::filesystem::exists(pathname);
    bool bLocalXMLExists = boost::filesystem::exists(xml_file);

    TiXmlDocument xml;
    if (!xml_data.empty()) // Parse remote XML data
    {
        std::cout << "XML: Using remote file" << std::endl;
        xml.Parse(xml_data.c_str());
    }
    else
    {   // Parse local XML data
        std::cout << "XML: Using local file" << std::endl;
        if (!bLocalXMLExists)
            std::cout << "XML: File doesn't exist (" << xml_file << ")" << std::endl;
        xml.LoadFile(xml_file);
    }

    // Check if file node exists in XML data
    TiXmlNode *fileNode = xml.FirstChild("file");
    if (!fileNode)
    {   // File node doesn't exist
        std::cout << "XML: Parsing failed / not valid XML" << std::endl;
        if (config.bDownload)
            bParsingFailed = true;
        else
            return res;
    }
    else
    {   // File node exists --> valid XML
        std::cout << "XML: Valid XML" << std::endl;
        TiXmlElement *fileElem = fileNode->ToElement();
        filename = fileElem->Attribute("name");
        filehash = fileElem->Attribute("md5");
        std::stringstream(fileElem->Attribute("chunks")) >> chunks;
        std::stringstream(fileElem->Attribute("total_size")) >> filesize;

        //Iterate through all chunk nodes
        TiXmlNode *chunkNode = fileNode->FirstChild();
        while (chunkNode)
        {
            TiXmlElement *chunkElem = chunkNode->ToElement();
            std::stringstream(chunkElem->Attribute("from")) >> from_offset;
            std::stringstream(chunkElem->Attribute("to")) >> to_offset;
            chunk_from.push_back(from_offset);
            chunk_to.push_back(to_offset);
            chunk_hash.push_back(chunkElem->GetText());
            chunkNode = fileNode->IterateChildren(chunkNode);
        }

        std::cout   << "XML: Parsing finished" << std::endl << std::endl
                    << filename << std::endl
                    << "\tMD5:\t" << filehash << std::endl
                    << "\tChunks:\t" << chunks << std::endl
                    << "\tSize:\t" << filesize << " bytes" << std::endl << std::endl;
    }

    // No local XML file and parsing failed.
    if (bParsingFailed && !bLocalXMLExists)
    {
        if (this->config.bDownload)
        {
            std::cout << "Downloading: " << filepath << std::endl;
            CURLcode result = this->downloadFile(url, filepath, xml_data);
            std::cout << std::endl;
            if  (
                    (!bFileExists && result == CURLE_OK) || /* File doesn't exist so only accept if everything was OK */
                    (bFileExists && (result == CURLE_OK || result == CURLE_RANGE_ERROR ))   /* File exists so accept also CURLE_RANGE_ERROR because curl will return CURLE_RANGE_ERROR */
                )                                                                           /* if the file is already fully downloaded and we want to resume it */
            {
                bLocalXMLExists = boost::filesystem::exists(xml_file); // Check to see if downloadFile saved XML data

                if (config.sXMLFile == "automatic" && !bLocalXMLExists)
                {
                    std::cout << "Starting automatic XML creation" << std::endl;
                    Util::createXML(filepath, config.iChunkSize, config.sXMLDirectory);
                }
                res = 1;
            }
        }
        else
        {
            std::cout << "Can't repair file." << std::endl;
        }
        return res;
    }

    // Check if file exists
    if (bFileExists)
    {
        // File exists
        if ((outfile = fopen(filepath.c_str(), "r+"))!=NULL )
        {
            fseek(outfile, 0, SEEK_END);
            offset = ftell(outfile);
        }
        else
        {
            std::cout << "Failed to open " << filepath << std::endl;
            return res;
        }
    }
    else
    {
        std::cout << "File doesn't exist " << filepath << std::endl;
        if (this->config.bDownload)
        {
            std::cout << "Downloading: " << filepath << std::endl;
            CURLcode result = this->downloadFile(url, filepath, xml_data);
            std::cout << std::endl;
            if (result == CURLE_OK)
            {
                if (config.sXMLFile == "automatic" && bParsingFailed)
                {
                    std::cout << "Starting automatic XML creation" << std::endl;
                    Util::createXML(filepath, config.iChunkSize, config.sXMLDirectory);
                }
                res = 1;
            }
        }
        return res;
    }

    if (offset != filesize)
    {
        std::cout   << "Filesizes don't match" << std::endl
                    << "Incomplete download or different version" << std::endl;
        fclose(outfile);
        if (this->config.bDownload)
        {
            std::cout << "Redownloading file" << std::endl;
            boost::filesystem::path path = filepath;
            if (!boost::filesystem::remove(path))
            {
                std::cout << "Failed to delete " << path << std::endl;
                res = 0;
            }
            else
            {
                CURLcode result = this->downloadFile(url, filepath, xml_data);
                std::cout << std::endl;
                if (result == CURLE_OK)
                    res = 1;
                else
                    res = 0;
            }
        }
        return res;
    }

    // Check all chunks
    int iChunksRepaired = 0;
    for (int i=0; i<chunks; i++)
    {
        size_t chunk_begin = chunk_from.at(i);
        size_t chunk_end = chunk_to.at(i);
        size_t size=0, chunk_size = chunk_end - chunk_begin + 1;
        std::string range = std::to_string(chunk_begin) + "-" + std::to_string(chunk_end); // Download range string for curl

        std::cout << "\033[0K\rChunk " << i << " (" << chunk_size << " bytes): ";
        fseek(outfile, chunk_begin, SEEK_SET);
        unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *));
        if (chunk == NULL)
        {
            std::cout << "Memory error" << std::endl;
            fclose(outfile);
            return res;
        }
        size = fread(chunk, 1, chunk_size, outfile);
        if (size != chunk_size)
        {
            std::cout << "Read error" << std::endl;
            free(chunk);
            fclose(outfile);
            return res;
        }
        std::string hash = Util::getChunkHash(chunk, chunk_size, RHASH_MD5);
        if (hash != chunk_hash.at(i))
        {
            std::cout << "Failed - downloading chunk" << std::endl;
            fseek(outfile, chunk_begin, SEEK_SET);
            curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
            curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, outfile);
            curl_easy_setopt(curlhandle, CURLOPT_RANGE, range.c_str()); //download range
            this->beginDownload(); //begin chunk download
            std::cout << std::endl;
            if (config.bReport)
                iChunksRepaired++;
            i--; //verify downloaded chunk
        }
        else
        {
            std::cout << "OK\r" << std::flush;
        }
        free(chunk);
        res = 1;
    }
    std::cout << std::endl;
    fclose(outfile);

    if (config.bReport)
    {
        std::string report_line = "Repaired [" + std::to_string(iChunksRepaired) + "/" + std::to_string(chunks) + "] " + filepath;
        this->report_ofs << report_line << std::endl;
    }

    return res;
}

// Download cover images
int Downloader::downloadCovers(const std::string& gamename, const std::string& directory, const std::string& cover_xml_data)
{
    int res = 0;
    TiXmlDocument xml;

    // Check that directory exists and create subdirectories
    boost::filesystem::path path = directory;
    if (boost::filesystem::exists(path))
    {
        if (!boost::filesystem::is_directory(path))
        {
            std::cout << path << " is not directory" << std::endl;
            return res;
        }

    }
    else
    {
        if (!boost::filesystem::create_directories(path))
        {
            std::cout << "Failed to create directory: " << path << std::endl;
            return res;
        }
    }

    xml.Parse(cover_xml_data.c_str());
    TiXmlElement *rootNode = xml.RootElement();
    if (!rootNode)
    {
        std::cout << "Not valid XML" << std::endl;
        return res;
    }
    else
    {
        TiXmlNode *gameNode = rootNode->FirstChild();
        while (gameNode)
        {
            TiXmlElement *gameElem = gameNode->ToElement();
            std::string game_name = gameElem->Attribute("name");

            if (game_name == gamename)
            {
                boost::match_results<std::string::const_iterator> what;
                TiXmlNode *coverNode = gameNode->FirstChild();
                while (coverNode)
                {
                    TiXmlElement *coverElem = coverNode->ToElement();
                    std::string cover_url = coverElem->GetText();
                    // Get file extension for the image
                    boost::regex e1(".*(\\.\\w+)$", boost::regex::perl | boost::regex::icase);
                    boost::regex_search(cover_url, what, e1);
                    std::string file_extension = what[1];
                    std::string cover_name = std::string("cover_") + coverElem->Attribute("id") + file_extension;
                    std::string filepath = directory + "/" + cover_name;

                    std::cout << "Downloading cover " << filepath << std::endl;
                    CURLcode result = this->downloadFile(cover_url, filepath);
                    std::cout << std::endl;
                    if (result == CURLE_OK)
                        res = 1;
                    else
                        res = 0;

                    if (result == CURLE_HTTP_RETURNED_ERROR)
                    {
                        long int response_code = 0;
                        result = curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code);
                        std::cout << "HTTP ERROR: ";
                        if (result == CURLE_OK)
                            std::cout << response_code << std::endl;
                        else
                            std::cout << "failed to get error code: " << curl_easy_strerror(result) << std::endl;
                    }

                    coverNode = gameNode->IterateChildren(coverNode);
                }
                break; // Found cover for game, no need to go through rest of the game nodes
            }
            gameNode = rootNode->IterateChildren(gameNode);
        }
    }

    return res;
}

CURLcode Downloader::beginDownload()
{
    this->timer.reset();
    CURLcode result = curl_easy_perform(curlhandle);
    this->resume_position = 0;
    return result;
}

std::string Downloader::getResponse(const std::string& url)
{
    std::ostringstream memory;

    curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
    curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeMemoryCallback);
    curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
    CURLcode result = curl_easy_perform(curlhandle);
    curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData);
    curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0);
    std::string response = memory.str();
    memory.str(std::string());

    if (result != CURLE_OK)
        std::cout << curl_easy_strerror(result) << std::endl;

    return response;
}

int Downloader::progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
{
    // on entry: dltotal - how much remains to download till the end of the file (bytes)
    //           dlnow   - how much was downloaded from the start of the program (bytes)
    unsigned int bar_length  = 26;
    Downloader* downloader = static_cast<Downloader*>(clientp);

    double rate; //  average download speed in B/s
    // trying to get rate and setting to NaN if it fails
    if (CURLE_OK != curl_easy_getinfo(downloader->curlhandle, CURLINFO_SPEED_DOWNLOAD, &rate))
       rate = std::numeric_limits<double>::quiet_NaN();

    // (Shmerl): this flag is needed to catch the case before anything was downloaded on resume,
    // and there is no way to calculate the fraction, so we set to 0 (otherwise it'd be 1).
    // This is to prevent the progress bar from jumping to 100% and then to lower value.
    // It's visually better to jump from 0% to higher one.
    bool starting = ((0.0 == dlnow) && (0.0 == dltotal));

    // (Shmerl): DEBUG: strange thing - when resuming a file which is already downloaded, dlnow is correctly 0.0
    // but dltotal is 389.0! This messes things up in the progress bar not showing the very last bar as full.
    // enable this debug line to test the problem:
    //
    //   printf("\r\033[0K dlnow: %0.2f, dltotal: %0.2f\r", dlnow, dltotal); fflush(stdout); return 0;
    //
    // For now making a quirky workaround and setting dltotal to 0.0 in that case.
    // It's probably better to find a real fix.
    if ((0.0 == dlnow) && (389.0 == dltotal)) dltotal = 0.0;

    // setting full dlwnow and dltotal
    double offset = static_cast<double>(downloader->getResumePosition());
    if (offset>0)
    {
        dlnow   += offset;
        dltotal += offset;
    }

    // Update progress bar every 100ms
    if (downloader->timer.getTimeBetweenUpdates()>=100 || dlnow == dltotal)
    {
        downloader->timer.reset();
        bptime::time_duration eta(bptime::seconds((long)((dltotal - dlnow) / rate)));
        std::stringstream eta_ss;
        if (eta.hours() > 23)
        {
           eta_ss << eta.hours() / 24 << "d " <<
                     std::setfill('0') << std::setw(2) << eta.hours() % 24 << "h " <<
                     std::setfill('0') << std::setw(2) << eta.minutes() << "m " <<
                     std::setfill('0') << std::setw(2) << eta.seconds() << "s";
        }
        else if (eta.hours() > 0)
        {
           eta_ss << eta.hours() << "h " <<
                     std::setfill('0') << std::setw(2) << eta.minutes() << "m " <<
                     std::setfill('0') << std::setw(2) << eta.seconds() << "s";
        }
        else if (eta.minutes() > 0)
        {
           eta_ss << eta.minutes() << "m " <<
                     std::setfill('0') << std::setw(2) << eta.seconds() << "s";
        }
        else
        {
           eta_ss << eta.seconds() << "s";
        }

        // Create progressbar
        double fraction = starting ? 0.0 : dlnow / dltotal;

        // assuming that config is provided.
        printf("\033[0K\r%3.0f%% ", fraction * 100);
        downloader->progressbar->draw(bar_length, fraction);

        // Download rate unit conversion
        std::string rate_unit;
        if (rate > 1048576) // 1 MB
        {
            rate /= 1048576;
            rate_unit = "MB/s";
        }
        else
        {
            rate /= 1024;
            rate_unit = "kB/s";
        }
        printf(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s\r", dlnow/1024/1024, dltotal/1024/1024, rate, rate_unit.c_str(), eta_ss.str().c_str());
        fflush(stdout);
    }

    return 0;
}

size_t Downloader::writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp) {
    std::ostringstream *stream = (std::ostringstream*)userp;
    size_t count = size * nmemb;
    stream->write(ptr, count);
    return count;
}

size_t Downloader::writeData(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
    return fwrite(ptr, size, nmemb, stream);
}

size_t Downloader::readData(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
    return fread(ptr, size, nmemb, stream);
}

size_t Downloader::getResumePosition()
{
    return this->resume_position;
}

// Login to GOG website
int Downloader::HTTP_Login(const std::string& email, const std::string& password)
{
    int res = 0;
    std::string postdata;
    std::ostringstream memory;
    std::string buk;

    // Get "buk" for login form
    std::string json = this->getResponse("http://www.gog.com/user/ajax/?a=get");

    Json::Value root;
    Json::Reader *jsonparser = new Json::Reader;
    bool parsingSuccessful = jsonparser->parse(json, root);
    if (!parsingSuccessful)
    {
        #ifdef DEBUG
            std::cerr << "DEBUG INFO (Downloader::HTTP_Login)" << std::endl << json << std::endl;
        #endif
        std::cout << jsonparser->getFormatedErrorMessages();
        return res = 0;
    }
    #ifdef DEBUG
        std::cerr << "DEBUG INFO (Downloader::HTTP_Login)" << std::endl << root << std::endl;
    #endif
    buk = root["buk"].asString();

    //Create postdata - escape characters in email/password to support special characters
    postdata = "log_email=" + (std::string)curl_easy_escape(curlhandle, email.c_str(),email.size())
            + "&log_password=" + (std::string)curl_easy_escape(curlhandle, password.c_str(), password.size())
            + "&buk=" + (std::string)curl_easy_escape(curlhandle, buk.c_str(), buk.size());
    curl_easy_setopt(curlhandle, CURLOPT_URL, "https://secure.gog.com/login");
    curl_easy_setopt(curlhandle, CURLOPT_POST, 1);
    curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str());
    curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeMemoryCallback);
    curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory);
    curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1);
    curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 1);
    curl_easy_setopt(curlhandle, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
    CURLcode result = curl_easy_perform(curlhandle);
    memory.str(std::string());

    if (result != CURLE_OK)
    {
        // Expected to hit maximum amount of redirects so don't print error on it
        if (result != CURLE_TOO_MANY_REDIRECTS)
            std::cout << curl_easy_strerror(result) << std::endl;
    }

    curl_easy_setopt(curlhandle, CURLOPT_HTTPGET, 1);
    curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, -1);
    json = this->getResponse("http://www.gog.com/user/ajax/?a=get");

    parsingSuccessful = jsonparser->parse(json, root);
    if (!parsingSuccessful)
    {
        #ifdef DEBUG
            std::cerr << "DEBUG INFO (Downloader::HTTP_Login)" << std::endl << json << std::endl;
        #endif
        std::cout << jsonparser->getFormatedErrorMessages();
        return res = 0;
    }
    #ifdef DEBUG
        std::cerr << "DEBUG INFO (Downloader::HTTP_Login)" << std::endl << root << std::endl;
    #endif

    if (root["user"]["email"].asString() == email && (result == CURLE_OK || result == CURLE_TOO_MANY_REDIRECTS))
        res = 1; // Login successful
    else
        res = 0; // Login failed

    delete jsonparser;

    return res;
}

// Get list of games from account page
std::vector< std::pair<std::string,std::string> > Downloader::getGames()
{
    std::vector< std::pair<std::string,std::string> > games;
    Json::Value root;
    Json::Reader *jsonparser = new Json::Reader;
    int i = 1;
    std::string html = "";
    std::string page_html = "";

    do
    {
        std::string response = this->getResponse("https://secure.gog.com/en/account/ajax?a=gamesShelfMore&s=title&q=&t=0&p=" + std::to_string(i));

        // Parse JSON
        if (!jsonparser->parse(response, root))
        {
            #ifdef DEBUG
                std::cerr << "DEBUG INFO (Downloader::getGames)" << std::endl << response << std::endl;
            #endif
            std::cout << jsonparser->getFormatedErrorMessages();
            delete jsonparser;
            exit(1);
        }
        #ifdef DEBUG
            std::cerr << "DEBUG INFO (Downloader::getGames)" << std::endl << root << std::endl;
        #endif
        page_html = root["html"].asString();
        html += page_html;
        i++;
    } while (!page_html.empty());

    delete jsonparser;

    // Parse HTML to get game names
    htmlcxx::HTML::ParserDom parser;
    tree<htmlcxx::HTML::Node> dom = parser.parseTree(html);
    tree<htmlcxx::HTML::Node>::iterator it = dom.begin();
    tree<htmlcxx::HTML::Node>::iterator end = dom.end();
    for (; it != end; ++it)
    {
        if (it->tagName()=="div")
        {
            it->parseAttributes();
            std::string classname = it->attribute("class").second;
            if (classname=="shelf_game")
            {
                // Game name is contained in data-gameindex attribute
                std::string game = it->attribute("data-gameindex").second;
                std::string gameid = it->attribute("data-gameid").second;
                if (!game.empty() && !gameid.empty())
                    games.push_back(std::make_pair(game,gameid));
            }
        }
    }

    return games;
}

// Get list of free games
std::vector< std::pair<std::string,std::string> > Downloader::getFreeGames()
{
    Json::Value root;
    Json::Reader *jsonparser = new Json::Reader;
    std::vector< std::pair<std::string,std::string> > games;
    std::string json = this->getResponse("https://secure.gog.com/games/ajax?a=search&f={\"price\":[\"free\"],\"sort\":\"title\"}&p=1&t=all");

    // Parse JSON
    if (!jsonparser->parse(json, root))
    {
        #ifdef DEBUG
            std::cerr << "DEBUG INFO (Downloader::getFreeGames)" << std::endl << json << std::endl;
        #endif
        std::cout << jsonparser->getFormatedErrorMessages();
        delete jsonparser;
        exit(1);
    }
    #ifdef DEBUG
        std::cerr << "DEBUG INFO (Downloader::getFreeGames)" << std::endl << root << std::endl;
    #endif
    std::string html = root["result"]["html"].asString();
    delete jsonparser;

    // Parse HTML to get game names
    htmlcxx::HTML::ParserDom parser;
    tree<htmlcxx::HTML::Node> dom = parser.parseTree(html);
    tree<htmlcxx::HTML::Node>::iterator it = dom.begin();
    tree<htmlcxx::HTML::Node>::iterator end = dom.end();
    for (; it != end; ++it)
    {
        if (it->tagName()=="span")
        {
            it->parseAttributes();
            std::string classname = it->attribute("class").second;
            if (classname=="gog-price game-owned")
            {
                // Game name is contained in data-gameindex attribute
                std::string game = it->attribute("data-gameindex").second;
                std::string id = it->attribute("data-gameid").second;
                if (!game.empty() && !id.empty())
                    games.push_back(std::make_pair(game,id));
            }
        }
    }

    return games;
}

std::vector<gameFile> Downloader::getExtras(const std::string& gamename, const std::string& gameid)
{
    Json::Value root;
    Json::Reader *jsonparser = new Json::Reader;
    std::vector<gameFile> extras;

    std::string gameDataUrl = "https://secure.gog.com/en/account/ajax?a=gamesListDetails&g=" + gameid;
    std::string json = this->getResponse(gameDataUrl);
    // Parse JSON
    if (!jsonparser->parse(json, root))
    {
        #ifdef DEBUG
            std::cerr << "DEBUG INFO (Downloader::getExtras)" << std::endl << json << std::endl;
        #endif
        std::cout << jsonparser->getFormatedErrorMessages();
        delete jsonparser;
        exit(1);
    }
    #ifdef DEBUG
        std::cerr << "DEBUG INFO (Downloader::getExtras)" << std::endl << root << std::endl;
    #endif
    std::string html = root["details"]["html"].asString();
    delete jsonparser;

    htmlcxx::HTML::ParserDom parser;
    tree<htmlcxx::HTML::Node> dom = parser.parseTree(html);
    tree<htmlcxx::HTML::Node>::iterator it = dom.begin();
    tree<htmlcxx::HTML::Node>::iterator end = dom.end();
    for (; it != end; ++it)
    {
        if (it->tagName()=="a")
        {
            it->parseAttributes();
            std::string href = it->attribute("href").second;
            // Extra links https://secure.gog.com/downlink/file/gamename/id_number
            if (href.find("/downlink/file/" + gamename + "/")!=std::string::npos)
            {
                std::string id, name, path;
                id.assign(href.begin()+href.find_last_of("/")+1, href.end());

                // Get path from download link
                std::string url = gogAPI->getExtraLink(gamename, id);
                if (url.find("/extras/") != std::string::npos)
                {
                    path.assign(url.begin()+url.find("/extras/"), url.begin()+url.find_first_of("?"));
                    path = "/" + gamename + path;
                }
                else
                {
                    path.assign(url.begin()+url.find_last_of("/")+1, url.begin()+url.find_first_of("?"));
                    path = "/" + gamename + "/extras/" + path;
                }

                // Get name from path
                name.assign(path.begin()+path.find_last_of("/")+1,path.end());

                extras.push_back(
                                    gameFile (  false,
                                                id,
                                                name,
                                                path,
                                                std::string()
                                            )
                                 );
            }
        }
    }

    return extras;
}

void Downloader::checkOrphans()
{
    // Always check everything when checking for orphaned files
    config.bInstallers = true;
    config.bExtras = true;
    config.bPatches = true;
    config.bLanguagePacks = true;

    if (this->games.empty())
        this->getGameDetails();

    std::vector<std::string> orphans;
    for (unsigned int i = 0; i < games.size(); ++i)
    {
        std::cout << "Checking for orphaned files " << i+1 << " / " << games.size() << "\r" << std::flush;
        boost::filesystem::path path (config.sDirectory + games[i].gamename);
        std::vector<boost::filesystem::path> filepath_vector;

        try
        {
            if (boost::filesystem::exists(path))
            {
                if (boost::filesystem::is_directory(path))
                {
                    // Recursively iterate over files in directory
                    boost::filesystem::recursive_directory_iterator end_iter;
                    boost::filesystem::recursive_directory_iterator dir_iter(path);
                    while (dir_iter != end_iter)
                    {
                        if (boost::filesystem::is_regular_file(dir_iter->status()))
                        {
                            std::string filename = dir_iter->path().filename().string();
                            boost::regex expression(config.sOrphanRegex); // Limit to files matching the regex
                            boost::match_results<std::string::const_iterator> what;
                            if (boost::regex_search(filename, what, expression))
                                filepath_vector.push_back(dir_iter->path());
                        }
                        dir_iter++;
                    }
                }
            }
            else
                std::cout << path << " does not exist" << std::endl;
        }
        catch (const boost::filesystem::filesystem_error& ex)
        {
            std::cout << ex.what() << std::endl;
        }

        if (!filepath_vector.empty())
        {
            for (unsigned int j = 0; j < filepath_vector.size(); ++j)
            {
                bool bFoundFile = false; // Assume that the file is orphaned

                // Check installers
                for (unsigned int k = 0; k < games[i].installers.size(); ++k)
                {
                    if (games[i].installers[k].path.find(filepath_vector[j].filename().string()) != std::string::npos)
                    {
                        bFoundFile = true;
                        break;
                    }
                }
                if (!bFoundFile)
                {   // Check extras
                    for (unsigned int k = 0; k < games[i].extras.size(); ++k)
                    {
                        if (games[i].extras[k].path.find(filepath_vector[j].filename().string()) != std::string::npos)
                        {
                            bFoundFile = true;
                            break;
                        }
                    }
                }
                if (!bFoundFile)
                {   // Check patches
                    for (unsigned int k = 0; k < games[i].patches.size(); ++k)
                    {
                        if (games[i].patches[k].path.find(filepath_vector[j].filename().string()) != std::string::npos)
                        {
                            bFoundFile = true;
                            break;
                        }
                    }
                }
                if (!bFoundFile)
                {   // Check language packs
                    for (unsigned int k = 0; k < games[i].languagepacks.size(); ++k)
                    {
                        if (games[i].languagepacks[k].path.find(filepath_vector[j].filename().string()) != std::string::npos)
                        {
                            bFoundFile = true;
                            break;
                        }
                    }
                }
                if (!bFoundFile)
                    orphans.push_back(filepath_vector[j].string());
            }
        }
    }
    std::cout << std::endl;

    if (!orphans.empty())
    {
        for (unsigned int i = 0; i < orphans.size(); ++i)
        {
            std::cout << orphans[i] << std::endl;
        }
    }
    else
    {
        std::cout << "No orphaned files" << std::endl;
    }

    return;
}

// Check status of files
void Downloader::checkStatus()
{
    if (this->games.empty())
        this->getGameDetails();

    for (unsigned int i = 0; i < games.size(); ++i)
    {
        if (config.bInstallers)
        {
            for (unsigned int j = 0; j < games[i].installers.size(); ++j)
            {
                boost::filesystem::path filepath = Util::makeFilepath(config.sDirectory, games[i].installers[j].path, games[i].gamename);

                std::string remoteHash;
                std::string localHash;
                bool bHashOK = true; // assume hash OK
                size_t filesize;

                localHash = this->getLocalFileHash(filepath.string());
                remoteHash = this->getRemoteFileHash(games[i].gamename, games[i].installers[j].id);

                if (boost::filesystem::exists(filepath))
                {
                    filesize = boost::filesystem::file_size(filepath);

                    if (remoteHash != localHash)
                        bHashOK = false;

                    std::cout << (bHashOK ? "OK " : "MD5 ") << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
                }
                else
                {
                    std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl;
                }
            }
        }

        if (config.bExtras)
        {
            for (unsigned int j = 0; j < games[i].extras.size(); ++j)
            {
                boost::filesystem::path filepath = Util::makeFilepath(config.sDirectory, games[i].extras[j].path, games[i].gamename, config.bSubDirectories ? "extras" : "");

                std::string localHash = this->getLocalFileHash(filepath.string());
                size_t filesize;

                if (boost::filesystem::exists(filepath))
                {
                    filesize = boost::filesystem::file_size(filepath);
                    std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
                }
                else
                {
                    std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl;
                }
            }
        }

        if (config.bPatches)
        {
            for (unsigned int j = 0; j < games[i].patches.size(); ++j)
            {
                boost::filesystem::path filepath = Util::makeFilepath(config.sDirectory, games[i].patches[j].path, games[i].gamename, config.bSubDirectories ? "patches" : "");

                std::string localHash = this->getLocalFileHash(filepath.string());
                size_t filesize;

                if (boost::filesystem::exists(filepath))
                {
                    filesize = boost::filesystem::file_size(filepath);
                    std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
                }
                else
                {
                    std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl;
                }
            }
        }

        if (config.bLanguagePacks)
        {
            for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j)
            {
                boost::filesystem::path filepath = Util::makeFilepath(config.sDirectory, games[i].languagepacks[j].path, games[i].gamename, config.bSubDirectories ? "languagepacks" : "");

                std::string localHash = this->getLocalFileHash(filepath.string());
                size_t filesize;

                if (boost::filesystem::exists(filepath))
                {
                    filesize = boost::filesystem::file_size(filepath);
                    std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
                }
                else
                {
                    std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl;
                }
            }
        }
    }

    return;
}

std::string Downloader::getLocalFileHash(const std::string& filepath)
{
    std::string localHash;
    boost::filesystem::path path = filepath;
    boost::filesystem::path local_xml_file = config.sXMLDirectory + "/" + path.filename().string() + ".xml";
    if (boost::filesystem::exists(local_xml_file))
    {
        TiXmlDocument local_xml;
        local_xml.LoadFile(local_xml_file.string());
        TiXmlNode *fileNodeLocal = local_xml.FirstChild("file");
        if (fileNodeLocal)
        {
            TiXmlElement *fileElemLocal = fileNodeLocal->ToElement();
            localHash = fileElemLocal->Attribute("md5");
        }
    }
    else
    {
        if (boost::filesystem::exists(path))
        {
            localHash = Util::getFileHash(path.string(), RHASH_MD5);
        }
    }
    return localHash;
}

std::string Downloader::getRemoteFileHash(const std::string& gamename, const std::string& id)
{
    std::string remoteHash;
    std::string xml_data = gogAPI->getXML(gamename, id);
    if (!xml_data.empty())
    {
        TiXmlDocument remote_xml;
        remote_xml.Parse(xml_data.c_str());
        TiXmlNode *fileNodeRemote = remote_xml.FirstChild("file");
        if (fileNodeRemote)
        {
            TiXmlElement *fileElemRemote = fileNodeRemote->ToElement();
            remoteHash = fileElemRemote->Attribute("md5");
        }
    }
    return remoteHash;
}