From 22933d85020b0b1f4ff285fbfdfca940f52fe1c3 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 23 Aug 2019 13:20:09 +0200 Subject: [PATCH 1/3] VolumeVerifier: Add datfile parsing --- Source/Core/Common/CommonPaths.h | 1 + Source/Core/Common/FileUtil.cpp | 1 + Source/Core/Common/FileUtil.h | 1 + Source/Core/DiscIO/CMakeLists.txt | 1 + Source/Core/DiscIO/DiscIO.vcxproj | 3 + Source/Core/DiscIO/VolumeVerifier.cpp | 281 ++++++++++++++++-- Source/Core/DiscIO/VolumeVerifier.h | 63 +++- Source/Core/DolphinQt/Config/VerifyWidget.cpp | 37 ++- Source/Core/DolphinQt/Config/VerifyWidget.h | 5 + 9 files changed, 353 insertions(+), 40 deletions(-) diff --git a/Source/Core/Common/CommonPaths.h b/Source/Core/Common/CommonPaths.h index 2c1057b3fe..bec3fc0772 100644 --- a/Source/Core/Common/CommonPaths.h +++ b/Source/Core/Common/CommonPaths.h @@ -41,6 +41,7 @@ #define MAPS_DIR "Maps" #define CACHE_DIR "Cache" #define COVERCACHE_DIR "GameCovers" +#define REDUMPCACHE_DIR "Redump" #define SHADERCACHE_DIR "Shaders" #define STATESAVES_DIR "StateSaves" #define SCREENSHOTS_DIR "ScreenShots" diff --git a/Source/Core/Common/FileUtil.cpp b/Source/Core/Common/FileUtil.cpp index 75118bffac..3c85563b56 100644 --- a/Source/Core/Common/FileUtil.cpp +++ b/Source/Core/Common/FileUtil.cpp @@ -765,6 +765,7 @@ static void RebuildUserDirectories(unsigned int dir_index) s_user_paths[D_MAPS_IDX] = s_user_paths[D_USER_IDX] + MAPS_DIR DIR_SEP; s_user_paths[D_CACHE_IDX] = s_user_paths[D_USER_IDX] + CACHE_DIR DIR_SEP; s_user_paths[D_COVERCACHE_IDX] = s_user_paths[D_CACHE_IDX] + COVERCACHE_DIR DIR_SEP; + s_user_paths[D_REDUMPCACHE_IDX] = s_user_paths[D_CACHE_IDX] + REDUMPCACHE_DIR DIR_SEP; s_user_paths[D_SHADERCACHE_IDX] = s_user_paths[D_CACHE_IDX] + SHADERCACHE_DIR DIR_SEP; s_user_paths[D_SHADERS_IDX] = s_user_paths[D_USER_IDX] + SHADERS_DIR DIR_SEP; s_user_paths[D_STATESAVES_IDX] = s_user_paths[D_USER_IDX] + STATESAVES_DIR DIR_SEP; diff --git a/Source/Core/Common/FileUtil.h b/Source/Core/Common/FileUtil.h index 57e645c285..3cb65e093a 100644 --- a/Source/Core/Common/FileUtil.h +++ b/Source/Core/Common/FileUtil.h @@ -31,6 +31,7 @@ enum D_MAPS_IDX, D_CACHE_IDX, D_COVERCACHE_IDX, + D_REDUMPCACHE_IDX, D_SHADERCACHE_IDX, D_SHADERS_IDX, D_STATESAVES_IDX, diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index 4e5faa6cd4..ca5fdd3ddd 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -45,5 +45,6 @@ add_library(discio target_link_libraries(discio PRIVATE + pugixml ZLIB::ZLIB ) diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index 7ad3731fc6..1d150c4db3 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -94,6 +94,9 @@ {2e6c348c-c75c-4d94-8d1e-9c1fcbf3efe4} + + {38fee76f-f347-484b-949c-b4649381cffb} + diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index af875e42f8..b0d4e8c3c5 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -11,15 +11,19 @@ #include #include #include +#include #include #include #include +#include #include #include "Common/Align.h" #include "Common/Assert.h" +#include "Common/CommonPaths.h" #include "Common/CommonTypes.h" +#include "Common/FileUtil.h" #include "Common/Logging/Log.h" #include "Common/MsgHandler.h" #include "Common/StringUtil.h" @@ -39,6 +43,184 @@ namespace DiscIO { +void RedumpVerifier::Start(const Volume& volume) +{ + if (volume.GetVolumeType() == Platform::GameCubeDisc) + m_dat_filename = "gamecube.dat"; + else if (volume.GetVolumeType() == Platform::WiiDisc) + m_dat_filename = "wii.dat"; + else + m_result.status = Status::Error; + + // We use GetGameTDBID instead of GetGameID so that Datel discs will be represented by an empty + // string, which matches Redump not having any serials for Datel discs. + m_game_id = volume.GetGameTDBID(); + if (m_game_id.size() > 4) + m_game_id = m_game_id.substr(0, 4); + + m_revision = volume.GetRevision().value_or(0); + m_disc_number = volume.GetDiscNumber().value_or(0); + m_size = volume.GetSize(); + + m_future = std::async(std::launch::async, [this] { return ScanXML(); }); +} + +static u8 ParseHexDigit(char c) +{ + if (c < '0') + return 0; // Error + + if (c >= 'a') + c -= 'a' - 'A'; + if (c >= 'A') + c -= 'A' - ('9' + 1); + c -= '0'; + + if (c >= 0x10) + return 0; // Error + + return c; +} + +static std::vector ParseHash(const char* str) +{ + std::vector hash; + while (str[0] && str[1]) + { + hash.push_back(static_cast(ParseHexDigit(str[0]) * 0x10 + ParseHexDigit(str[1]))); + str += 2; + } + return hash; +} + +std::vector RedumpVerifier::ScanXML() +{ + const std::string path = File::GetUserPath(D_REDUMPCACHE_IDX) + DIR_SEP + m_dat_filename; + + pugi::xml_document doc; + { + std::string data; + if (!File::ReadFileToString(path, data) || !doc.load_buffer(data.data(), data.size())) + { + m_result = {Status::Error, Common::GetStringT("Failed to parse Redump.org data")}; + return {}; + } + } + + std::vector potential_matches; + const pugi::xml_node datafile = doc.child("datafile"); + for (const pugi::xml_node game : datafile.children("game")) + { + std::string version_string = game.child("version").text().as_string(); + + // Strip out prefix (e.g. "v1.02" -> "02", "Rev 2" -> "2") + const size_t last_non_numeric = version_string.find_last_not_of("0123456789"); + if (last_non_numeric != std::string::npos) + version_string = version_string.substr(last_non_numeric + 1); + + const int version = version_string.empty() ? 0 : std::stoi(version_string); + + // The revisions for Korean GameCube games whose four-char game IDs end in E are numbered from + // 0x30 in ring codes and in disc headers, but Redump switched to numbering them from 0 in 2019. + if (version % 0x30 != m_revision % 0x30) + continue; + + const std::string serials = game.child("serial").text().as_string(); + if (serials.empty()) + { + // This case is reached for Datel discs + if (!m_game_id.empty()) + continue; // Non-empty m_game_id means we're verifying a non-Datel disc + } + else + { + bool serial_match_found = false; + + // If a disc has multiple possible serials, they are delimited with ", ". We want to loop + // through all the serials until we find a match, because even though they normally only + // differ in the region code at the end (which we don't care about), there is an edge case + // disc with the game ID "G96P" and the serial "DL-DOL-D96P-EUR, DL-DOL-G96P-EUR". + for (const std::string& serial_str : SplitString(serials, ',')) + { + const std::string_view serial = StripSpaces(serial_str); + + // Skip the prefix, normally either "DL-DOL-" or "RVL-" (depending on the console), + // but there are some exceptions like the "RVLE-SBSE-USA-B0" serial. + const size_t first_dash = serial.find_first_of('-', 3); + const size_t game_id_start = + first_dash == std::string::npos ? std::string::npos : first_dash + 1; + + if (serial.size() < game_id_start + 4) + { + ERROR_LOG(DISCIO, "Invalid serial in redump datfile: %s", serial_str.c_str()); + continue; + } + + const std::string_view game_id = serial.substr(game_id_start, 4); + if (game_id != m_game_id) + continue; + + u8 disc_number = 0; + if (serial.size() > game_id_start + 5 && serial[game_id_start + 5] >= '0' && + serial[game_id_start + 5] <= '9') + { + disc_number = serial[game_id_start + 5] - '0'; + } + if (disc_number != m_disc_number) + continue; + + serial_match_found = true; + break; + } + if (!serial_match_found) + continue; + } + + PotentialMatch& potential_match = potential_matches.emplace_back(); + const pugi::xml_node rom = game.child("rom"); + potential_match.size = rom.attribute("size").as_ullong(); + potential_match.hashes.crc32 = ParseHash(rom.attribute("crc").value()); + potential_match.hashes.md5 = ParseHash(rom.attribute("md5").value()); + potential_match.hashes.sha1 = ParseHash(rom.attribute("sha1").value()); + } + + return potential_matches; +} + +static bool HashesMatch(const std::vector& calculated, const std::vector& expected) +{ + return calculated.empty() || calculated == expected; +} + +RedumpVerifier::Result RedumpVerifier::Finish(const Hashes>& hashes) +{ + if (m_result.status == Status::Error) + return m_result; + + if (hashes.crc32.empty() && hashes.md5.empty() && hashes.sha1.empty()) + return m_result; + + const std::vector potential_matches = m_future.get(); + for (PotentialMatch p : potential_matches) + { + if (HashesMatch(hashes.crc32, p.hashes.crc32) && HashesMatch(hashes.md5, p.hashes.md5) && + HashesMatch(hashes.sha1, p.hashes.sha1) && m_size == p.size) + { + return {Status::GoodDump, Common::GetStringT("Good dump")}; + } + } + + // We only return bad dump if there's a disc that we know this dump should match but that doesn't + // match. For disc without IDs (i.e. Datel discs), we don't have a good way of knowing whether we + // have a bad dump or just a dump that isn't in Redump, so we always pick unknown instead of bad + // dump for those to be on the safe side. (Besides, it's possible to dump a Datel disc correctly + // and have it not match Redump if you don't use the same replacement value for bad sectors.) + if (!potential_matches.empty() && !m_game_id.empty()) + return {Status::BadDump, Common::GetStringT("Bad dump")}; + + return {Status::Unknown, Common::GetStringT("Unknown disc")}; +} + constexpr u64 MINI_DVD_SIZE = 1459978240; // GameCube constexpr u64 SL_DVD_SIZE = 4699979776; // Wii retail constexpr u64 SL_DVD_R_SIZE = 4707319808; // Wii RVT-R @@ -47,12 +229,16 @@ constexpr u64 DL_DVD_R_SIZE = 8543666176; // Wii RVT-R constexpr u64 BLOCK_SIZE = 0x20000; -VolumeVerifier::VolumeVerifier(const Volume& volume, Hashes hashes_to_calculate) - : m_volume(volume), m_hashes_to_calculate(hashes_to_calculate), +VolumeVerifier::VolumeVerifier(const Volume& volume, bool redump_verification, + Hashes hashes_to_calculate) + : m_volume(volume), m_redump_verification(redump_verification), + m_hashes_to_calculate(hashes_to_calculate), m_calculating_any_hash(hashes_to_calculate.crc32 || hashes_to_calculate.md5 || hashes_to_calculate.sha1), m_max_progress(volume.GetSize()) { + if (!m_calculating_any_hash) + m_redump_verification = false; } VolumeVerifier::~VolumeVerifier() = default; @@ -62,6 +248,9 @@ void VolumeVerifier::Start() ASSERT(!m_started); m_started = true; + if (m_redump_verification) + m_redump_verifier.Start(m_volume); + m_is_tgc = m_volume.GetBlobType() == BlobType::TGC; m_is_datel = IsDisc(m_volume.GetVolumeType()) && !GetBootDOLOffset(m_volume, m_volume.GetGamePartition()).has_value(); @@ -934,6 +1123,26 @@ void VolumeVerifier::Finish() const Severity highest_severity = m_result.problems.empty() ? Severity::None : m_result.problems[0].severity; + if (m_redump_verification) + m_result.redump = m_redump_verifier.Finish(m_result.hashes); + + if (m_result.redump.status == RedumpVerifier::Status::GoodDump || + (m_volume.GetVolumeType() == Platform::WiiWAD && !m_is_not_retail && + m_result.problems.empty())) + { + if (m_result.problems.empty()) + { + m_result.summary_text = Common::GetStringT("This is a good dump."); + } + else + { + m_result.summary_text = + Common::GetStringT("This is a good dump according to Redump.org, but Dolphin has found " + "problems. This might be a bug in Dolphin."); + } + return; + } + if (m_is_datel) { m_result.summary_text = Common::GetStringT("Dolphin is unable to verify unlicensed discs."); @@ -948,35 +1157,49 @@ void VolumeVerifier::Finish() return; } - switch (highest_severity) + if (m_result.redump.status == RedumpVerifier::Status::BadDump && + highest_severity <= Severity::Low) { - case Severity::None: - if (IsWii(m_volume.GetVolumeType()) && !m_is_not_retail) - { - m_result.summary_text = Common::GetStringT( - "No problems were found. This does not guarantee that this is a good dump, " - "but since Wii titles contain a lot of verification data, it does mean that " - "there most likely are no problems that will affect emulation."); - } - else - { - m_result.summary_text = Common::GetStringT("No problems were found."); - } - break; - case Severity::Low: - m_result.summary_text = - Common::GetStringT("Problems with low severity were found. They will most " - "likely not prevent the game from running."); - break; - case Severity::Medium: - m_result.summary_text = - Common::GetStringT("Problems with medium severity were found. The whole game " - "or certain parts of the game might not work correctly."); - break; - case Severity::High: m_result.summary_text = Common::GetStringT( - "Problems with high severity were found. The game will most likely not work at all."); - break; + "This is a bad dump. This doesn't necessarily mean that the game won't run correctly."); + } + else + { + if (m_result.redump.status == RedumpVerifier::Status::BadDump) + { + m_result.summary_text = Common::GetStringT("This is a bad dump.") + "\n\n"; + } + + switch (highest_severity) + { + case Severity::None: + if (IsWii(m_volume.GetVolumeType()) && !m_is_not_retail) + { + m_result.summary_text = Common::GetStringT( + "No problems were found. This does not guarantee that this is a good dump, " + "but since Wii titles contain a lot of verification data, it does mean that " + "there most likely are no problems that will affect emulation."); + } + else + { + m_result.summary_text = Common::GetStringT("No problems were found."); + } + break; + case Severity::Low: + m_result.summary_text = + Common::GetStringT("Problems with low severity were found. They will most " + "likely not prevent the game from running."); + break; + case Severity::Medium: + m_result.summary_text += + Common::GetStringT("Problems with medium severity were found. The whole game " + "or certain parts of the game might not work correctly."); + break; + case Severity::High: + m_result.summary_text += Common::GetStringT( + "Problems with high severity were found. The game will most likely not work at all."); + break; + } } if (m_volume.GetVolumeType() == Platform::GameCubeDisc) diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index b6bec4ac13..0b908811f5 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -21,7 +21,7 @@ // To be used as follows: // -// VolumeVerifier verifier(volume); +// VolumeVerifier verifier(volume, redump_verification, hashes_to_calculate); // verifier.Start(); // while (verifier.GetBytesProcessed() != verifier.GetTotalBytes()) // verifier.Process(); @@ -36,6 +36,53 @@ namespace DiscIO { class FileInfo; +template +struct Hashes +{ + T crc32; + T md5; + T sha1; +}; + +class RedumpVerifier final +{ +public: + enum class Status + { + Unknown, + GoodDump, + BadDump, + Error, + }; + + struct Result + { + Status status = Status::Unknown; + std::string message; + }; + + void Start(const Volume& volume); + Result Finish(const Hashes>& hashes); + +private: + struct PotentialMatch + { + u64 size; + Hashes> hashes; + }; + + std::vector ScanXML(); + + std::string m_dat_filename; + std::string m_game_id; + u16 m_revision; + u8 m_disc_number; + u64 m_size; + + std::future> m_future; + Result m_result; +}; + class VolumeVerifier final { public: @@ -53,22 +100,15 @@ public: std::string text; }; - template - struct Hashes - { - T crc32; - T md5; - T sha1; - }; - struct Result { Hashes> hashes; std::string summary_text; std::vector problems; + RedumpVerifier::Result redump; }; - VolumeVerifier(const Volume& volume, Hashes hashes_to_calculate); + VolumeVerifier(const Volume& volume, bool redump_verification, Hashes hashes_to_calculate); ~VolumeVerifier(); void Start(); @@ -111,6 +151,9 @@ private: bool m_is_datel = false; bool m_is_not_retail = false; + bool m_redump_verification; + RedumpVerifier m_redump_verifier; + Hashes m_hashes_to_calculate{}; bool m_calculating_any_hash = false; unsigned long m_crc32_context = 0; diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.cpp b/Source/Core/DolphinQt/Config/VerifyWidget.cpp index 488343fc76..f688c38932 100644 --- a/Source/Core/DolphinQt/Config/VerifyWidget.cpp +++ b/Source/Core/DolphinQt/Config/VerifyWidget.cpp @@ -29,6 +29,7 @@ VerifyWidget::VerifyWidget(std::shared_ptr volume) : m_volume(st layout->addWidget(m_problems); layout->addWidget(m_summary_text); layout->addLayout(m_hash_layout); + layout->addLayout(m_redump_layout); layout->addWidget(m_verify_button); layout->setStretchFactor(m_problems, 5); @@ -55,8 +56,21 @@ void VerifyWidget::CreateWidgets() std::tie(m_md5_checkbox, m_md5_line_edit) = AddHashLine(m_hash_layout, tr("MD5:")); std::tie(m_sha1_checkbox, m_sha1_line_edit) = AddHashLine(m_hash_layout, tr("SHA-1:")); + m_redump_layout = new QFormLayout; + if (DiscIO::IsDisc(m_volume->GetVolumeType())) + { + std::tie(m_redump_checkbox, m_redump_line_edit) = + AddHashLine(m_redump_layout, tr("Redump.org Status:")); + } + else + { + m_redump_checkbox = nullptr; + m_redump_line_edit = nullptr; + } + // Extend line edits to their maximum possible widths (needed on macOS) m_hash_layout->setFieldGrowthPolicy(QFormLayout::AllNonFixedFieldsGrow); + m_redump_layout->setFieldGrowthPolicy(QFormLayout::AllNonFixedFieldsGrow); m_verify_button = new QPushButton(tr("Verify Integrity"), this); } @@ -80,6 +94,9 @@ std::pair VerifyWidget::AddHashLine(QFormLayout* layout, void VerifyWidget::ConnectWidgets() { connect(m_verify_button, &QPushButton::clicked, this, &VerifyWidget::Verify); + + connect(m_md5_checkbox, &QCheckBox::stateChanged, this, &VerifyWidget::UpdateRedumpEnabled); + connect(m_sha1_checkbox, &QCheckBox::stateChanged, this, &VerifyWidget::UpdateRedumpEnabled); } static void SetHash(QLineEdit* line_edit, const std::vector& hash) @@ -89,10 +106,25 @@ static void SetHash(QLineEdit* line_edit, const std::vector& hash) line_edit->setText(QString::fromLatin1(byte_array.toHex())); } +bool VerifyWidget::CanVerifyRedump() const +{ + // We don't allow Redump verification with CRC32 only since generating a collision is too easy + return m_md5_checkbox->isChecked() || m_sha1_checkbox->isChecked(); +} + +void VerifyWidget::UpdateRedumpEnabled() +{ + if (m_redump_checkbox) + m_redump_checkbox->setEnabled(CanVerifyRedump()); +} + void VerifyWidget::Verify() { + const bool redump_verification = + CanVerifyRedump() && m_redump_checkbox && m_redump_checkbox->isChecked(); + DiscIO::VolumeVerifier verifier( - *m_volume, + *m_volume, redump_verification, {m_crc32_checkbox->isChecked(), m_md5_checkbox->isChecked(), m_sha1_checkbox->isChecked()}); // We have to divide the number of processed bytes with something so it won't make ints overflow @@ -147,6 +179,9 @@ void VerifyWidget::Verify() SetHash(m_crc32_line_edit, result.hashes.crc32); SetHash(m_md5_line_edit, result.hashes.md5); SetHash(m_sha1_line_edit, result.hashes.sha1); + + if (m_redump_line_edit) + m_redump_line_edit->setText(QString::fromStdString(result.redump.message)); } void VerifyWidget::SetProblemCellText(int row, int column, QString text) diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.h b/Source/Core/DolphinQt/Config/VerifyWidget.h index c2433cbd49..c5c6945c4f 100644 --- a/Source/Core/DolphinQt/Config/VerifyWidget.h +++ b/Source/Core/DolphinQt/Config/VerifyWidget.h @@ -32,6 +32,8 @@ private: std::pair AddHashLine(QFormLayout* layout, QString text); void ConnectWidgets(); + bool CanVerifyRedump() const; + void UpdateRedumpEnabled(); void Verify(); void SetProblemCellText(int row, int column, QString text); @@ -39,11 +41,14 @@ private: QTableWidget* m_problems; QTextEdit* m_summary_text; QFormLayout* m_hash_layout; + QFormLayout* m_redump_layout; QCheckBox* m_crc32_checkbox; QCheckBox* m_md5_checkbox; QCheckBox* m_sha1_checkbox; + QCheckBox* m_redump_checkbox; QLineEdit* m_crc32_line_edit; QLineEdit* m_md5_line_edit; QLineEdit* m_sha1_line_edit; + QLineEdit* m_redump_line_edit; QPushButton* m_verify_button; }; From 3eb360b8185f8f0284e3f67de1bd3d9d8225e98d Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 23 Aug 2019 22:38:44 +0200 Subject: [PATCH 2/3] VolumeVerifier: Add zip support for datfile --- Source/Core/Common/CMakeLists.txt | 2 + Source/Core/Common/Common.vcxproj | 1 + Source/Core/Common/Common.vcxproj.filters | 1 + Source/Core/Common/MinizipUtil.h | 42 ++++++++++++++++ Source/Core/DiscIO/CMakeLists.txt | 1 + Source/Core/DiscIO/VolumeVerifier.cpp | 50 ++++++++++++++----- Source/Core/DiscIO/VolumeVerifier.h | 5 +- .../UICommon/ResourcePack/ResourcePack.cpp | 36 ++----------- 8 files changed, 92 insertions(+), 46 deletions(-) create mode 100644 Source/Core/Common/MinizipUtil.h diff --git a/Source/Core/Common/CMakeLists.txt b/Source/Core/Common/CMakeLists.txt index c7752540df..0dd69b6c43 100644 --- a/Source/Core/Common/CMakeLists.txt +++ b/Source/Core/Common/CMakeLists.txt @@ -82,6 +82,7 @@ add_library(common MemArena.h MemoryUtil.cpp MemoryUtil.h + MinizipUtil.h MsgHandler.cpp MsgHandler.h NandPaths.cpp @@ -134,6 +135,7 @@ PUBLIC enet fmt::fmt ${MBEDTLS_LIBRARIES} + minizip PRIVATE ${CURL_LIBRARIES} diff --git a/Source/Core/Common/Common.vcxproj b/Source/Core/Common/Common.vcxproj index 4fd4119418..f1b56e99f1 100644 --- a/Source/Core/Common/Common.vcxproj +++ b/Source/Core/Common/Common.vcxproj @@ -139,6 +139,7 @@ + diff --git a/Source/Core/Common/Common.vcxproj.filters b/Source/Core/Common/Common.vcxproj.filters index 5849e34f66..6a0261032f 100644 --- a/Source/Core/Common/Common.vcxproj.filters +++ b/Source/Core/Common/Common.vcxproj.filters @@ -59,6 +59,7 @@ + diff --git a/Source/Core/Common/MinizipUtil.h b/Source/Core/Common/MinizipUtil.h new file mode 100644 index 0000000000..8d8bc7d692 --- /dev/null +++ b/Source/Core/Common/MinizipUtil.h @@ -0,0 +1,42 @@ +// Copyright 2019 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include + +#include + +#include "Common/CommonTypes.h" +#include "Common/ScopeGuard.h" + +namespace Common +{ +// Reads all of the current file. destination must be big enough to fit the whole file. +template +bool ReadFileFromZip(unzFile file, ContiguousContainer* destination) +{ + const u32 MAX_BUFFER_SIZE = 65535; + + if (unzOpenCurrentFile(file) != UNZ_OK) + return false; + + Common::ScopeGuard guard{[&] { unzCloseCurrentFile(file); }}; + + u32 bytes_to_go = static_cast(destination->size()); + while (bytes_to_go > 0) + { + const int bytes_read = + unzReadCurrentFile(file, &(*destination)[destination->size() - bytes_to_go], + std::min(bytes_to_go, MAX_BUFFER_SIZE)); + + if (bytes_read < 0) + return false; + + bytes_to_go -= static_cast(bytes_read); + } + + return true; +} +} // namespace Common diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index ca5fdd3ddd..bfeab923fb 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -45,6 +45,7 @@ add_library(discio target_link_libraries(discio PRIVATE + minizip pugixml ZLIB::ZLIB ) diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index b0d4e8c3c5..de100bfb76 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -25,7 +26,9 @@ #include "Common/CommonTypes.h" #include "Common/FileUtil.h" #include "Common/Logging/Log.h" +#include "Common/MinizipUtil.h" #include "Common/MsgHandler.h" +#include "Common/ScopeGuard.h" #include "Common/StringUtil.h" #include "Common/Swap.h" #include "Core/IOS/Device.h" @@ -46,9 +49,9 @@ namespace DiscIO void RedumpVerifier::Start(const Volume& volume) { if (volume.GetVolumeType() == Platform::GameCubeDisc) - m_dat_filename = "gamecube.dat"; + m_platform = "gc"; else if (volume.GetVolumeType() == Platform::WiiDisc) - m_dat_filename = "wii.dat"; + m_platform = "wii"; else m_result.status = Status::Error; @@ -62,7 +65,35 @@ void RedumpVerifier::Start(const Volume& volume) m_disc_number = volume.GetDiscNumber().value_or(0); m_size = volume.GetSize(); - m_future = std::async(std::launch::async, [this] { return ScanXML(); }); + m_future = std::async(std::launch::async, [this] { return ScanDatfile(ReadDatfile()); }); +} + +std::vector RedumpVerifier::ReadDatfile() +{ + const std::string path = File::GetUserPath(D_REDUMPCACHE_IDX) + DIR_SEP + m_platform + ".zip"; + + unzFile file = unzOpen(path.c_str()); + if (!file) + return {}; + + Common::ScopeGuard file_guard{[&] { unzClose(file); }}; + + // Check that the zip file contains exactly one file + if (unzGoToFirstFile(file) != UNZ_OK) + return {}; + if (unzGoToNextFile(file) != UNZ_END_OF_LIST_OF_FILE) + return {}; + + // Read the file + if (unzGoToFirstFile(file) != UNZ_OK) + return {}; + unz_file_info file_info; + unzGetCurrentFileInfo(file, &file_info, nullptr, 0, nullptr, 0, nullptr, 0); + std::vector data(file_info.uncompressed_size); + if (!Common::ReadFileFromZip(file, &data)) + return {}; + + return data; } static u8 ParseHexDigit(char c) @@ -93,18 +124,13 @@ static std::vector ParseHash(const char* str) return hash; } -std::vector RedumpVerifier::ScanXML() +std::vector RedumpVerifier::ScanDatfile(const std::vector& data) { - const std::string path = File::GetUserPath(D_REDUMPCACHE_IDX) + DIR_SEP + m_dat_filename; - pugi::xml_document doc; + if (!doc.load_buffer(data.data(), data.size())) { - std::string data; - if (!File::ReadFileToString(path, data) || !doc.load_buffer(data.data(), data.size())) - { - m_result = {Status::Error, Common::GetStringT("Failed to parse Redump.org data")}; - return {}; - } + m_result = {Status::Error, Common::GetStringT("Failed to parse Redump.org data")}; + return {}; } std::vector potential_matches; diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 0b908811f5..d10c5c97ff 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -71,9 +71,10 @@ private: Hashes> hashes; }; - std::vector ScanXML(); + std::vector ReadDatfile(); + std::vector ScanDatfile(const std::vector& data); - std::string m_dat_filename; + std::string m_platform; std::string m_game_id; u16 m_revision; u8 m_disc_number; diff --git a/Source/Core/UICommon/ResourcePack/ResourcePack.cpp b/Source/Core/UICommon/ResourcePack/ResourcePack.cpp index 24c6f7c76e..b0587f3ca5 100644 --- a/Source/Core/UICommon/ResourcePack/ResourcePack.cpp +++ b/Source/Core/UICommon/ResourcePack/ResourcePack.cpp @@ -10,6 +10,7 @@ #include "Common/FileSearch.h" #include "Common/FileUtil.h" +#include "Common/MinizipUtil.h" #include "Common/ScopeGuard.h" #include "Common/StringUtil.h" @@ -20,35 +21,6 @@ namespace ResourcePack { constexpr char TEXTURE_PATH[] = "Load/Textures/"; -// Since minzip doesn't provide a way to unzip a file of a length > 65535, we have to implement -// this ourselves -template -static bool ReadCurrentFileUnlimited(unzFile file, ContiguousContainer& destination) -{ - const u32 MAX_BUFFER_SIZE = 65535; - - if (unzOpenCurrentFile(file) != UNZ_OK) - return false; - - Common::ScopeGuard guard{[&] { unzCloseCurrentFile(file); }}; - - auto bytes_to_go = static_cast(destination.size()); - while (bytes_to_go > 0) - { - const int bytes_read = unzReadCurrentFile(file, &destination[destination.size() - bytes_to_go], - std::min(bytes_to_go, MAX_BUFFER_SIZE)); - - if (bytes_read < 0) - { - return false; - } - - bytes_to_go -= static_cast(bytes_read); - } - - return true; -} - ResourcePack::ResourcePack(const std::string& path) : m_path(path) { auto file = unzOpen(path.c_str()); @@ -72,7 +44,7 @@ ResourcePack::ResourcePack(const std::string& path) : m_path(path) unzGetCurrentFileInfo(file, &manifest_info, nullptr, 0, nullptr, 0, nullptr, 0); std::string manifest_contents(manifest_info.uncompressed_size, '\0'); - if (!ReadCurrentFileUnlimited(file, manifest_contents)) + if (!Common::ReadFileFromZip(file, &manifest_contents)) { m_valid = false; m_error = "Failed to read manifest.json"; @@ -96,7 +68,7 @@ ResourcePack::ResourcePack(const std::string& path) : m_path(path) m_logo_data.resize(logo_info.uncompressed_size); - if (!ReadCurrentFileUnlimited(file, m_logo_data)) + if (!Common::ReadFileFromZip(file, &m_logo_data)) { m_valid = false; m_error = "Failed to read logo.png"; @@ -208,7 +180,7 @@ bool ResourcePack::Install(const std::string& path) unzGetCurrentFileInfo(file, &texture_info, nullptr, 0, nullptr, 0, nullptr, 0); std::vector data(texture_info.uncompressed_size); - if (!ReadCurrentFileUnlimited(file, data)) + if (!Common::ReadFileFromZip(file, &data)) { m_error = "Failed to read texture " + texture; return false; From 87c5e0b30e8f9174c96dc9b78e02863e940c5372 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 24 Aug 2019 12:53:13 +0200 Subject: [PATCH 3/3] VolumeVerifier: Add Redump.org downloading --- Source/Core/DiscIO/VolumeVerifier.cpp | 109 +++++++++++++++++++++++--- Source/Core/DiscIO/VolumeVerifier.h | 22 +++++- 2 files changed, 118 insertions(+), 13 deletions(-) diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index de100bfb76..fdc5c1d487 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -24,13 +24,16 @@ #include "Common/Assert.h" #include "Common/CommonPaths.h" #include "Common/CommonTypes.h" +#include "Common/File.h" #include "Common/FileUtil.h" +#include "Common/HttpRequest.h" #include "Common/Logging/Log.h" #include "Common/MinizipUtil.h" #include "Common/MsgHandler.h" #include "Common/ScopeGuard.h" #include "Common/StringUtil.h" #include "Common/Swap.h" +#include "Common/Version.h" #include "Core/IOS/Device.h" #include "Core/IOS/ES/ES.h" #include "Core/IOS/ES/Formats.h" @@ -46,15 +49,11 @@ namespace DiscIO { +RedumpVerifier::DownloadState RedumpVerifier::m_gc_download_state; +RedumpVerifier::DownloadState RedumpVerifier::m_wii_download_state; + void RedumpVerifier::Start(const Volume& volume) { - if (volume.GetVolumeType() == Platform::GameCubeDisc) - m_platform = "gc"; - else if (volume.GetVolumeType() == Platform::WiiDisc) - m_platform = "wii"; - else - m_result.status = Status::Error; - // We use GetGameTDBID instead of GetGameID so that Datel discs will be represented by an empty // string, which matches Redump not having any serials for Datel discs. m_game_id = volume.GetGameTDBID(); @@ -65,14 +64,102 @@ void RedumpVerifier::Start(const Volume& volume) m_disc_number = volume.GetDiscNumber().value_or(0); m_size = volume.GetSize(); - m_future = std::async(std::launch::async, [this] { return ScanDatfile(ReadDatfile()); }); + const DiscIO::Platform platform = volume.GetVolumeType(); + + m_future = std::async(std::launch::async, [this, platform]() -> std::vector { + std::string system; + DownloadState* download_state; + switch (platform) + { + case Platform::GameCubeDisc: + system = "gc"; + download_state = &m_gc_download_state; + break; + + case Platform::WiiDisc: + system = "wii"; + download_state = &m_wii_download_state; + break; + + default: + m_result.status = Status::Error; + return {}; + } + + DownloadStatus status; + { + std::lock_guard lk(download_state->mutex); + download_state->status = DownloadDatfile(system, download_state->status); + status = download_state->status; + } + + switch (download_state->status) + { + case DownloadStatus::FailButOldCacheAvailable: + ERROR_LOG(DISCIO, "Failed to fetch data from Redump.org, using old cached data instead"); + [[fallthrough]]; + case DownloadStatus::Success: + return ScanDatfile(ReadDatfile(system)); + + case DownloadStatus::SystemNotAvailable: + m_result = {Status::Error, Common::GetStringT("Wii data is not public yet")}; + return {}; + + case DownloadStatus::Fail: + default: + m_result = {Status::Error, Common::GetStringT("Failed to connect to Redump.org")}; + return {}; + } + }); } -std::vector RedumpVerifier::ReadDatfile() +static std::string GetPathForSystem(const std::string& system) { - const std::string path = File::GetUserPath(D_REDUMPCACHE_IDX) + DIR_SEP + m_platform + ".zip"; + return File::GetUserPath(D_REDUMPCACHE_IDX) + DIR_SEP + system + ".zip"; +} - unzFile file = unzOpen(path.c_str()); +RedumpVerifier::DownloadStatus RedumpVerifier::DownloadDatfile(const std::string& system, + DownloadStatus old_status) +{ + if (old_status == DownloadStatus::Success || old_status == DownloadStatus::SystemNotAvailable) + return old_status; + + Common::HttpRequest request; + + const std::optional> result = + request.Get("http://redump.org/datfile/" + system + "/serial,version", + {{"User-Agent", Common::scm_rev_str}}); + + const std::string output_path = GetPathForSystem(system); + + if (!result) + { + return File::Exists(output_path) ? DownloadStatus::FailButOldCacheAvailable : + DownloadStatus::Fail; + } + + if (result->size() > 1 && (*result)[0] == '<' && (*result)[1] == '!') + { + // This is an HTML page, not a zip file like we want + + if (File::Exists(output_path)) + return DownloadStatus::FailButOldCacheAvailable; + + const std::string system_not_available_message = "System \"" + system + "\" doesn't exist."; + const bool system_not_available_match = + result->end() != std::search(result->begin(), result->end(), + system_not_available_message.begin(), + system_not_available_message.end()); + return system_not_available_match ? DownloadStatus::SystemNotAvailable : DownloadStatus::Fail; + } + + File::IOFile(output_path, "wb").WriteBytes(result->data(), result->size()); + return DownloadStatus::Success; +} + +std::vector RedumpVerifier::ReadDatfile(const std::string& system) +{ + unzFile file = unzOpen(GetPathForSystem(system).c_str()); if (!file) return {}; diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index d10c5c97ff..b2f447db91 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -65,16 +65,31 @@ public: Result Finish(const Hashes>& hashes); private: + enum class DownloadStatus + { + NotAttempted, + Success, + Fail, + FailButOldCacheAvailable, + SystemNotAvailable, + }; + + struct DownloadState + { + std::mutex mutex; + DownloadStatus status = DownloadStatus::NotAttempted; + }; + struct PotentialMatch { u64 size; Hashes> hashes; }; - std::vector ReadDatfile(); + static DownloadStatus DownloadDatfile(const std::string& system, DownloadStatus old_status); + static std::vector ReadDatfile(const std::string& system); std::vector ScanDatfile(const std::vector& data); - std::string m_platform; std::string m_game_id; u16 m_revision; u8 m_disc_number; @@ -82,6 +97,9 @@ private: std::future> m_future; Result m_result; + + static DownloadState m_gc_download_state; + static DownloadState m_wii_download_state; }; class VolumeVerifier final