From 8a9597e32e98707d5bcdc1a41d8c45e3174d4047 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 4 Apr 2020 20:56:20 +0200 Subject: [PATCH] DiscIO: Allow converting from formats other than ISO and GCZ The constant DESIRED_BUFFER_SIZE was determined by multiplying the old hardcoded value 32 with the default GCZ block size 16 KiB. Not sure if it actually is the best value, but it seems fine. --- Source/Core/DiscIO/Blob.h | 14 +-- Source/Core/DiscIO/CISOBlob.h | 5 +- Source/Core/DiscIO/CompressedBlob.cpp | 94 ++++++++++----------- Source/Core/DiscIO/CompressedBlob.h | 1 + Source/Core/DiscIO/DiscScrubber.cpp | 35 +------- Source/Core/DiscIO/DiscScrubber.h | 7 +- Source/Core/DiscIO/DriveBlob.h | 5 ++ Source/Core/DiscIO/VolumeVerifier.cpp | 2 +- Source/Core/DiscIO/WbfsBlob.h | 3 + Source/Core/DolphinQt/GameList/GameList.cpp | 10 +-- 10 files changed, 80 insertions(+), 96 deletions(-) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 52b083ab63..9cb00087d2 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -41,11 +41,16 @@ class BlobReader { public: virtual ~BlobReader() {} + virtual BlobType GetBlobType() const = 0; + virtual u64 GetRawSize() const = 0; virtual u64 GetDataSize() const = 0; virtual bool IsDataSizeAccurate() const = 0; + // Returns 0 if the format does not use blocks + virtual u64 GetBlockSize() const { return 0; } + // NOT thread-safe - can't call this from multiple threads. virtual bool Read(u64 offset, u64 size, u8* out_ptr) = 0; template @@ -160,10 +165,9 @@ std::unique_ptr CreateBlobReader(const std::string& filename); typedef bool (*CompressCB)(const std::string& text, float percent, void* arg); -bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path, - u32 sub_type = 0, int sector_size = 16384, CompressCB callback = nullptr, - void* arg = nullptr); -bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type = 0, + int sector_size = 16384, CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback = nullptr, void* arg = nullptr); } // namespace DiscIO diff --git a/Source/Core/DiscIO/CISOBlob.h b/Source/Core/DiscIO/CISOBlob.h index dc606bb4a5..66f950f074 100644 --- a/Source/Core/DiscIO/CISOBlob.h +++ b/Source/Core/DiscIO/CISOBlob.h @@ -37,12 +37,15 @@ public: static std::unique_ptr Create(File::IOFile file); BlobType GetBlobType() const override { return BlobType::CISO; } + + u64 GetRawSize() const override; // The CISO format does not save the original file size. // This function returns an upper bound. u64 GetDataSize() const override; bool IsDataSizeAccurate() const override { return false; } - u64 GetRawSize() const override; + u64 GetBlockSize() const override { return m_block_size; } + bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 5248170683..33790ad3b0 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -17,6 +17,7 @@ #include #include +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/FileUtil.h" @@ -153,24 +154,20 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr) return true; } -bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path, - u32 sub_type, int block_size, CompressCB callback, void* arg) +bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type, + int block_size, CompressCB callback, void* arg) { bool scrubbing = false; - File::IOFile infile(infile_path, "rb"); - if (IsGCZBlob(infile)) - { - PanicAlertT("\"%s\" is already compressed! Cannot compress it further.", infile_path.c_str()); - return false; - } - + std::unique_ptr infile = CreateDisc(infile_path); if (!infile) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(infile->IsSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -182,11 +179,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi } DiscScrubber disc_scrubber; - std::unique_ptr volume; if (sub_type == 1) { - volume = CreateDisc(infile_path); - if (!volume || !disc_scrubber.SetupScrub(volume.get(), block_size)) + if (!disc_scrubber.SetupScrub(infile.get())) { PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", infile_path.c_str()); @@ -206,7 +201,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi header.magic_cookie = GCZ_MAGIC; header.sub_type = sub_type; header.block_size = block_size; - header.data_size = infile.GetSize(); + header.data_size = infile->GetSize(); // round upwards! header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size); @@ -220,10 +215,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi outfile.Seek(sizeof(CompressedBlobHeader), SEEK_CUR); // seek past the offset and hash tables (we will write them at the end) outfile.Seek((sizeof(u64) + sizeof(u32)) * header.num_blocks, SEEK_CUR); - // seek to the start of the input file to make sure we get everything - infile.Seek(0, SEEK_SET); // Now we are ready to write compressed data! + u64 inpos = 0; u64 position = 0; int num_compressed = 0; int num_stored = 0; @@ -234,7 +228,6 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi { if (i % progress_monitor == 0) { - const u64 inpos = infile.Tell(); int ratio = 0; if (inpos != 0) ratio = (int)(100 * position / inpos); @@ -252,13 +245,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi offsets[i] = position; - size_t read_bytes; - if (scrubbing) - read_bytes = disc_scrubber.GetNextBlock(infile, in_buf.data()); - else - infile.ReadArray(in_buf.data(), header.block_size, &read_bytes); - if (read_bytes < header.block_size) - std::fill(in_buf.begin() + read_bytes, in_buf.begin() + header.block_size, 0); + const u64 bytes_to_read = scrubbing && disc_scrubber.CanBlockBeScrubbed(inpos) ? + 0 : + std::min(block_size, header.data_size - inpos); + + success = infile->Read(inpos, bytes_to_read, in_buf.data(), PARTITION_NONE); + if (!success) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + break; + } + + std::fill(in_buf.begin() + bytes_to_read, in_buf.begin() + header.block_size, 0); int retval = deflateReset(&z); z.next_in = in_buf.data(); @@ -305,6 +303,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi break; } + inpos += block_size; position += write_size; hashes[i] = Common::HashAdler32(write_buf, write_size); @@ -337,27 +336,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi return success; } -bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback, void* arg) +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback, void* arg) { - std::unique_ptr reader; - { - File::IOFile infile(infile_path, "rb"); - if (!IsGCZBlob(infile)) - { - PanicAlertT("File not compressed"); - return false; - } - - reader = CompressedBlobReader::Create(std::move(infile), infile_path); - } - + std::unique_ptr reader = CreateBlobReader(infile_path); if (!reader) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(reader->IsDataSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -368,11 +358,20 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out return false; } - const CompressedBlobHeader& header = reader->GetHeader(); - static const size_t BUFFER_BLOCKS = 32; - size_t buffer_size = header.block_size * BUFFER_BLOCKS; + constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; + u64 buffer_size = reader->GetBlockSize(); + if (buffer_size == 0) + { + buffer_size = DESIRED_BUFFER_SIZE; + } + else + { + while (buffer_size < DESIRED_BUFFER_SIZE) + buffer_size *= 2; + } + std::vector buffer(buffer_size); - u32 num_buffers = (header.num_blocks + BUFFER_BLOCKS - 1) / BUFFER_BLOCKS; + const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; int progress_monitor = std::max(1, num_buffers / 100); bool success = true; @@ -389,8 +388,13 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out } } const u64 inpos = i * buffer_size; - const u64 sz = std::min(buffer_size, header.data_size - inpos); - reader->Read(inpos, sz, buffer.data()); + const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); + if (!reader->Read(inpos, sz, buffer.data())) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + success = false; + break; + } if (!outfile.WriteBytes(buffer.data(), sz)) { PanicAlertT("Failed to write the output file \"%s\".\n" @@ -407,10 +411,6 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out outfile.Close(); File::Delete(outfile_path); } - else - { - outfile.Resize(header.data_size); - } return success; } diff --git a/Source/Core/DiscIO/CompressedBlob.h b/Source/Core/DiscIO/CompressedBlob.h index 0e290542c6..8c63aa68d6 100644 --- a/Source/Core/DiscIO/CompressedBlob.h +++ b/Source/Core/DiscIO/CompressedBlob.h @@ -52,6 +52,7 @@ public: u64 GetRawSize() const override { return m_file_size; } u64 GetDataSize() const override { return m_header.data_size; } bool IsDataSizeAccurate() const override { return true; } + u64 GetBlockSize() const override { return m_header.block_size; } u64 GetBlockCompressedSize(u64 block_num) const; bool GetBlock(u64 block_num, u8* out_ptr) override; diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index 8a06d9266e..07d504d72a 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -14,6 +14,7 @@ #include #include "Common/Align.h" +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/Logging/Log.h" @@ -29,19 +30,11 @@ constexpr size_t CLUSTER_SIZE = 0x8000; DiscScrubber::DiscScrubber() = default; DiscScrubber::~DiscScrubber() = default; -bool DiscScrubber::SetupScrub(const Volume* disc, int block_size) +bool DiscScrubber::SetupScrub(const Volume* disc) { if (!disc) return false; m_disc = disc; - m_block_size = block_size; - - if (CLUSTER_SIZE % m_block_size != 0) - { - ERROR_LOG(DISCIO, "Block size %u is not a factor of 0x8000, scrubbing not possible", - m_block_size); - return false; - } m_file_size = m_disc->GetSize(); @@ -54,34 +47,10 @@ bool DiscScrubber::SetupScrub(const Volume* disc, int block_size) // Fill out table of free blocks const bool success = ParseDisc(); - m_block_count = 0; - m_is_scrubbing = success; return success; } -size_t DiscScrubber::GetNextBlock(File::IOFile& in, u8* buffer) -{ - const u64 current_offset = m_block_count * m_block_size; - - size_t read_bytes = 0; - if (CanBlockBeScrubbed(current_offset)) - { - DEBUG_LOG(DISCIO, "Freeing 0x%016" PRIx64, current_offset); - std::fill(buffer, buffer + m_block_size, 0x00); - in.Seek(m_block_size, SEEK_CUR); - read_bytes = m_block_size; - } - else - { - DEBUG_LOG(DISCIO, "Used 0x%016" PRIx64, current_offset); - in.ReadArray(buffer, m_block_size, &read_bytes); - } - - m_block_count++; - return read_bytes; -} - bool DiscScrubber::CanBlockBeScrubbed(u64 offset) const { return m_is_scrubbing && m_free_table[offset / CLUSTER_SIZE]; diff --git a/Source/Core/DiscIO/DiscScrubber.h b/Source/Core/DiscIO/DiscScrubber.h index 15e0cda7f7..b3e7e7a9a0 100644 --- a/Source/Core/DiscIO/DiscScrubber.h +++ b/Source/Core/DiscIO/DiscScrubber.h @@ -34,8 +34,9 @@ public: DiscScrubber(); ~DiscScrubber(); - bool SetupScrub(const Volume* disc, int block_size); - size_t GetNextBlock(File::IOFile& in, u8* buffer); + bool SetupScrub(const Volume* disc); + + // Returns true if the specified 32 KiB block only contains unused data bool CanBlockBeScrubbed(u64 offset) const; private: @@ -72,8 +73,6 @@ private: std::vector m_free_table; u64 m_file_size = 0; - u64 m_block_count = 0; - u32 m_block_size = 0; bool m_is_scrubbing = false; }; diff --git a/Source/Core/DiscIO/DriveBlob.h b/Source/Core/DiscIO/DriveBlob.h index fddfbee7a2..46c141b1d3 100644 --- a/Source/Core/DiscIO/DriveBlob.h +++ b/Source/Core/DiscIO/DriveBlob.h @@ -23,11 +23,15 @@ class DriveReader : public SectorReader public: static std::unique_ptr Create(const std::string& drive); ~DriveReader(); + BlobType GetBlobType() const override { return BlobType::DRIVE; } + u64 GetRawSize() const override { return m_size; } u64 GetDataSize() const override { return m_size; } bool IsDataSizeAccurate() const override { return true; } + u64 GetBlockSize() const override { return ECC_BLOCK_SIZE; } + private: DriveReader(const std::string& drive); bool GetBlock(u64 block_num, u8* out_ptr) override; @@ -41,6 +45,7 @@ private: File::IOFile m_file; bool IsOK() const { return m_file.IsOpen() && m_file.IsGood(); } #endif + static constexpr u64 ECC_BLOCK_SIZE = 0x8000; u64 m_size = 0; }; diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 949457cd84..556d40c793 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -1031,7 +1031,7 @@ void VolumeVerifier::SetUpHashing() else if (m_volume.GetVolumeType() == Platform::WiiDisc) { // Set up a DiscScrubber for checking whether blocks with errors are unused - m_scrubber.SetupScrub(&m_volume, VolumeWii::BLOCK_TOTAL_SIZE); + m_scrubber.SetupScrub(&m_volume); } std::sort(m_blocks.begin(), m_blocks.end(), diff --git a/Source/Core/DiscIO/WbfsBlob.h b/Source/Core/DiscIO/WbfsBlob.h index 36de02f62d..0283b1c98b 100644 --- a/Source/Core/DiscIO/WbfsBlob.h +++ b/Source/Core/DiscIO/WbfsBlob.h @@ -24,6 +24,7 @@ public: static std::unique_ptr Create(File::IOFile file, const std::string& path); BlobType GetBlobType() const override { return BlobType::WBFS; } + u64 GetRawSize() const override { return m_size; } // The WBFS format does not save the original file size. // This function returns a constant upper bound @@ -31,6 +32,8 @@ public: u64 GetDataSize() const override; bool IsDataSizeAccurate() const override { return false; } + u64 GetBlockSize() const override { return m_wbfs_sector_size; } + bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DolphinQt/GameList/GameList.cpp b/Source/Core/DolphinQt/GameList/GameList.cpp index a635bee291..e4bf1c47bf 100644 --- a/Source/Core/DolphinQt/GameList/GameList.cpp +++ b/Source/Core/DolphinQt/GameList/GameList.cpp @@ -595,8 +595,8 @@ void GameList::CompressISO(bool decompress) } good = std::async(std::launch::async, [&] { - const bool good = DiscIO::DecompressBlobToFile(original_path, dst_path.toStdString(), - &CompressCB, &progress_dialog); + const bool good = DiscIO::ConvertToPlain(original_path, dst_path.toStdString(), &CompressCB, + &progress_dialog); progress_dialog.Reset(); return good; }); @@ -612,9 +612,9 @@ void GameList::CompressISO(bool decompress) good = std::async(std::launch::async, [&] { const bool good = - DiscIO::CompressFileToBlob(original_path, dst_path.toStdString(), - file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, - 16384, &CompressCB, &progress_dialog); + DiscIO::ConvertToGCZ(original_path, dst_path.toStdString(), + file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, + &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; });