From a8d9d466c9e61bad4ef0f741c4f690982c69abb0 Mon Sep 17 00:00:00 2001 From: Sude Date: Fri, 1 Mar 2019 14:09:52 +0200 Subject: [PATCH] Make progress info show estimated total remaining size and time --- include/gamedetails.h | 1 + include/util.h | 3 + src/downloader.cpp | 165 +++++++++++++++--------------------------- src/gamedetails.cpp | 28 +++++++ src/util.cpp | 39 ++++++++++ 5 files changed, 130 insertions(+), 106 deletions(-) diff --git a/include/gamedetails.h b/include/gamedetails.h index 6ab0ad2..4a47ac9 100644 --- a/include/gamedetails.h +++ b/include/gamedetails.h @@ -38,6 +38,7 @@ class gameDetails std::string getChangelogFilepath(); Json::Value getDetailsAsJson(); std::vector getGameFileVector(); + std::vector getGameFileVectorFiltered(const unsigned int& iType); virtual ~gameDetails(); protected: void filterListWithPriorities(std::vector& list, const gameSpecificConfig& config); diff --git a/include/util.h b/include/util.h index 982564d..4fc3c60 100644 --- a/include/util.h +++ b/include/util.h @@ -22,6 +22,7 @@ #include #include #include +#include struct gameItem { @@ -73,6 +74,8 @@ namespace Util void shortenStringToTerminalWidth(std::string& str); std::string getJsonUIntValueAsString(const Json::Value& json); std::string getStrippedString(std::string str); + std::string makeEtaString(const unsigned long long& iBytesRemaining, const double& dlRate); + std::string makeEtaString(const boost::posix_time::time_duration& duration); template std::string formattedString(const std::string& format, Args ... args) { diff --git a/src/downloader.cpp b/src/downloader.cpp index 44614b1..c20381e 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -47,6 +48,7 @@ ThreadSafeQueue gameDetailsQueue; ThreadSafeQueue dlQueueGalaxy; ThreadSafeQueue dlQueueGalaxy_MojoSetupHack; std::mutex mtx_create_directories; // Mutex for creating directories in Downloader::processDownloadQueue +std::atomic iTotalRemainingBytes(0); static curl_off_t WriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp) { @@ -862,34 +864,6 @@ void Downloader::download() this->saveChangelog(games[i].changelog, filepath); } - if (conf.dlConf.bInstallers) - { - for (unsigned int j = 0; j < games[i].installers.size(); ++j) - { - dlQueue.push(games[i].installers[j]); - } - } - if (conf.dlConf.bPatches) - { - for (unsigned int j = 0; j < games[i].patches.size(); ++j) - { - dlQueue.push(games[i].patches[j]); - } - } - if (conf.dlConf.bExtras) - { - for (unsigned int j = 0; j < games[i].extras.size(); ++j) - { - dlQueue.push(games[i].extras[j]); - } - } - if (conf.dlConf.bLanguagePacks) - { - for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j) - { - dlQueue.push(games[i].languagepacks[j]); - } - } if (conf.dlConf.bDLC && !games[i].dlcs.empty()) { for (unsigned int j = 0; j < games[i].dlcs.size(); ++j) @@ -904,37 +878,25 @@ void Downloader::download() std::string filepath = games[i].dlcs[j].getChangelogFilepath(); this->saveChangelog(games[i].dlcs[j].changelog, filepath); } - - if (conf.dlConf.bInstallers) - { - for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k) - { - dlQueue.push(games[i].dlcs[j].installers[k]); - } - } - if (conf.dlConf.bPatches) - { - for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k) - { - dlQueue.push(games[i].dlcs[j].patches[k]); - } - } - if (conf.dlConf.bExtras) - { - for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k) - { - dlQueue.push(games[i].dlcs[j].extras[k]); - } - } - if (conf.dlConf.bLanguagePacks) - { - for (unsigned int k = 0; k < games[i].dlcs[j].languagepacks.size(); ++k) - { - dlQueue.push(games[i].dlcs[j].languagepacks[k]); - } - } } } + + auto vFiles = games[i].getGameFileVectorFiltered(conf.dlConf.iInclude); + for (auto gf : vFiles) + { + dlQueue.push(gf); + unsigned long long filesize = 0; + try + { + filesize = std::stoll(gf.size); + } + catch (std::invalid_argument& e) + { + filesize = 0; + } + iTotalRemainingBytes.fetch_add(filesize); + } + } if (!dlQueue.empty()) @@ -1634,30 +1596,7 @@ int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t d rate = (size_last - size_first) / static_cast((time_last - time_first)); } - bptime::time_duration eta(bptime::seconds((long)((dltotal - dlnow) / rate))); - std::stringstream eta_ss; - if (eta.hours() > 23) - { - eta_ss << eta.hours() / 24 << "d " << - std::setfill('0') << std::setw(2) << eta.hours() % 24 << "h " << - std::setfill('0') << std::setw(2) << eta.minutes() << "m " << - std::setfill('0') << std::setw(2) << eta.seconds() << "s"; - } - else if (eta.hours() > 0) - { - eta_ss << eta.hours() << "h " << - std::setfill('0') << std::setw(2) << eta.minutes() << "m " << - std::setfill('0') << std::setw(2) << eta.seconds() << "s"; - } - else if (eta.minutes() > 0) - { - eta_ss << eta.minutes() << "m " << - std::setfill('0') << std::setw(2) << eta.seconds() << "s"; - } - else - { - eta_ss << eta.seconds() << "s"; - } + std::string etastring = Util::makeEtaString((dltotal - dlnow), rate); // Create progressbar double fraction = starting ? 0.0 : static_cast(dlnow) / static_cast(dltotal); @@ -1676,7 +1615,7 @@ int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t d rate /= 1024; rate_unit = "kB/s"; } - std::string status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s\r", static_cast(dlnow)/1024/1024, static_cast(dltotal)/1024/1024, rate, rate_unit.c_str(), eta_ss.str().c_str()); + std::string status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s\r", static_cast(dlnow)/1024/1024, static_cast(dltotal)/1024/1024, rate, rate_unit.c_str(), etastring.c_str()); int status_text_length = status_text.length() + 6; if ((status_text_length + bar_length) > iTermWidth) @@ -2596,6 +2535,17 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); + unsigned long long filesize = 0; + try + { + filesize = std::stoll(gf.size); + } + catch (std::invalid_argument& e) + { + filesize = 0; + } + iTotalRemainingBytes.fetch_sub(filesize); + // Get directory from filepath boost::filesystem::path filepath = gf.getFilepath(); filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); @@ -2990,6 +2940,7 @@ template void Downloader::printProgress(const ThreadSafeQueue& d int iTermWidth = Util::getTerminalWidth(); double total_rate = 0; + bptime::time_duration eta_total_seconds; // Create progress info text for all download threads std::vector vProgressText; @@ -3019,29 +2970,8 @@ template void Downloader::printProgress(const ThreadSafeQueue& d int progress_percentage_text_length = progress_percentage_text.length() + 1; bptime::time_duration eta(bptime::seconds((long)((progress_info.dltotal - progress_info.dlnow) / progress_info.rate))); - std::stringstream eta_ss; - if (eta.hours() > 23) - { - eta_ss << eta.hours() / 24 << "d " << - std::setfill('0') << std::setw(2) << eta.hours() % 24 << "h " << - std::setfill('0') << std::setw(2) << eta.minutes() << "m " << - std::setfill('0') << std::setw(2) << eta.seconds() << "s"; - } - else if (eta.hours() > 0) - { - eta_ss << eta.hours() << "h " << - std::setfill('0') << std::setw(2) << eta.minutes() << "m " << - std::setfill('0') << std::setw(2) << eta.seconds() << "s"; - } - else if (eta.minutes() > 0) - { - eta_ss << eta.minutes() << "m " << - std::setfill('0') << std::setw(2) << eta.seconds() << "s"; - } - else - { - eta_ss << eta.seconds() << "s"; - } + eta_total_seconds += eta; + std::string etastring = Util::makeEtaString(eta); std::string rate_unit; if (progress_info.rate > 1048576) // 1 MB @@ -3055,7 +2985,7 @@ template void Downloader::printProgress(const ThreadSafeQueue& d rate_unit = "kB/s"; } - std::string progress_status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s", static_cast(progress_info.dlnow)/1024/1024, static_cast(progress_info.dltotal)/1024/1024, progress_info.rate, rate_unit.c_str(), eta_ss.str().c_str()); + std::string progress_status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s", static_cast(progress_info.dlnow)/1024/1024, static_cast(progress_info.dltotal)/1024/1024, progress_info.rate, rate_unit.c_str(), etastring.c_str()); int status_text_length = progress_status_text.length() + 1; if ((status_text_length + progress_percentage_text_length + bar_length) > iTermWidth) @@ -3077,6 +3007,16 @@ template void Downloader::printProgress(const ThreadSafeQueue& d // Total download speed and number of remaining tasks in download queue if (dl_status != DLSTATUS_FINISHED) { + unsigned long long total_remaining = iTotalRemainingBytes.load(); + std::string total_eta_str; + if (total_remaining > 0) + { + bptime::time_duration eta(bptime::seconds((long)(total_remaining / total_rate))); + eta += eta_total_seconds; + total_eta_str = Util::makeEtaString(eta); + total_eta_str = Util::formattedString(" (%0.2fMB) ETA: %s", static_cast(total_remaining)/1024/1024, total_eta_str.c_str()); + } + std::ostringstream ss; if (Globals::globalConfig.iThreads > 1) { @@ -3094,6 +3034,10 @@ template void Downloader::printProgress(const ThreadSafeQueue& d ss << "Total: " << std::setprecision(2) << std::fixed << total_rate << rate_unit << " | "; } ss << "Remaining: " << download_queue.size(); + + if (!total_eta_str.empty()) + ss << total_eta_str; + vProgressText.push_back(ss.str()); } @@ -3381,6 +3325,7 @@ void Downloader::galaxyInstallGame(const std::string& product_id, int build_inde std::cout << "\tmd5: " << items[i].md5 << std::endl; } totalSize += items[i].totalSizeUncompressed; + iTotalRemainingBytes.fetch_add(items[i].totalSizeCompressed); dlQueueGalaxy.push(items[i]); } @@ -3467,6 +3412,7 @@ void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Con while (dlQueueGalaxy.try_pop(item)) { vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); + iTotalRemainingBytes.fetch_sub(item.totalSizeCompressed); boost::filesystem::path path = install_path + "/" + item.path; @@ -4067,11 +4013,17 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id) // Add files to download queue for (std::uintmax_t i = 0; i < vZipFiles.size(); ++i) + { dlQueueGalaxy_MojoSetupHack.push(vZipFiles[i]); + iTotalRemainingBytes.fetch_add(vZipFiles[i].comp_size); + } // Add symlinks to download queue for (std::uintmax_t i = 0; i < vZipFilesSymlink.size(); ++i) + { dlQueueGalaxy_MojoSetupHack.push(vZipFilesSymlink[i]); + iTotalRemainingBytes.fetch_add(vZipFilesSymlink[i].comp_size); + } // Limit thread count to number of items in download queue unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlQueueGalaxy_MojoSetupHack.size())); @@ -4138,6 +4090,7 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns while (dlQueueGalaxy_MojoSetupHack.try_pop(zfe)) { vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); + iTotalRemainingBytes.fetch_sub(zfe.comp_size); boost::filesystem::path path = zfe.filepath; boost::filesystem::path path_tmp = zfe.filepath + ".lgogdltmp"; diff --git a/src/gamedetails.cpp b/src/gamedetails.cpp index 473e0ee..fb8b421 100644 --- a/src/gamedetails.cpp +++ b/src/gamedetails.cpp @@ -210,3 +210,31 @@ std::vector gameDetails::getGameFileVector() return vGameFiles; } + +// Return vector containing all game files matching download filters +std::vector gameDetails::getGameFileVectorFiltered(const unsigned int& iType) +{ + std::vector vGameFiles = this->getGameFileVector(); + + std::remove_if( + vGameFiles.begin(), + vGameFiles.end(), + [iType](gameFile gf) + { + bool bRemove = false; + if (gf.type & iType) + { + // Remove if DLC but DLCs not enabled + if ( !((iType & GFTYPE_DLC) & iType) ) + bRemove = true; + } + else + { + bRemove = true; + } + return bRemove; + } + ); + + return vGameFiles; +} diff --git a/src/util.cpp b/src/util.cpp index f40aba7..4c0f25c 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -629,3 +629,42 @@ std::string Util::getStrippedString(std::string str) ); return str; } + +std::string Util::makeEtaString(const unsigned long long& iBytesRemaining, const double& dlRate) +{ + boost::posix_time::time_duration duration(boost::posix_time::seconds((long)(iBytesRemaining / dlRate))); + + return Util::makeEtaString(duration); +} + +std::string Util::makeEtaString(const boost::posix_time::time_duration& duration) +{ + std::string etastr; + std::stringstream eta_ss; + + if (duration.hours() > 23) + { + eta_ss << duration.hours() / 24 << "d " << + std::setfill('0') << std::setw(2) << duration.hours() % 24 << "h " << + std::setfill('0') << std::setw(2) << duration.minutes() << "m " << + std::setfill('0') << std::setw(2) << duration.seconds() << "s"; + } + else if (duration.hours() > 0) + { + eta_ss << duration.hours() << "h " << + std::setfill('0') << std::setw(2) << duration.minutes() << "m " << + std::setfill('0') << std::setw(2) << duration.seconds() << "s"; + } + else if (duration.minutes() > 0) + { + eta_ss << duration.minutes() << "m " << + std::setfill('0') << std::setw(2) << duration.seconds() << "s"; + } + else + { + eta_ss << duration.seconds() << "s"; + } + etastr = eta_ss.str(); + + return etastr; +}