diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 52b083ab63..9cb00087d2 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -41,11 +41,16 @@ class BlobReader { public: virtual ~BlobReader() {} + virtual BlobType GetBlobType() const = 0; + virtual u64 GetRawSize() const = 0; virtual u64 GetDataSize() const = 0; virtual bool IsDataSizeAccurate() const = 0; + // Returns 0 if the format does not use blocks + virtual u64 GetBlockSize() const { return 0; } + // NOT thread-safe - can't call this from multiple threads. virtual bool Read(u64 offset, u64 size, u8* out_ptr) = 0; template @@ -160,10 +165,9 @@ std::unique_ptr CreateBlobReader(const std::string& filename); typedef bool (*CompressCB)(const std::string& text, float percent, void* arg); -bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path, - u32 sub_type = 0, int sector_size = 16384, CompressCB callback = nullptr, - void* arg = nullptr); -bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type = 0, + int sector_size = 16384, CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback = nullptr, void* arg = nullptr); } // namespace DiscIO diff --git a/Source/Core/DiscIO/CISOBlob.h b/Source/Core/DiscIO/CISOBlob.h index dc606bb4a5..66f950f074 100644 --- a/Source/Core/DiscIO/CISOBlob.h +++ b/Source/Core/DiscIO/CISOBlob.h @@ -37,12 +37,15 @@ public: static std::unique_ptr Create(File::IOFile file); BlobType GetBlobType() const override { return BlobType::CISO; } + + u64 GetRawSize() const override; // The CISO format does not save the original file size. // This function returns an upper bound. u64 GetDataSize() const override; bool IsDataSizeAccurate() const override { return false; } - u64 GetRawSize() const override; + u64 GetBlockSize() const override { return m_block_size; } + bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 5248170683..33790ad3b0 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -17,6 +17,7 @@ #include #include +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/FileUtil.h" @@ -153,24 +154,20 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr) return true; } -bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path, - u32 sub_type, int block_size, CompressCB callback, void* arg) +bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type, + int block_size, CompressCB callback, void* arg) { bool scrubbing = false; - File::IOFile infile(infile_path, "rb"); - if (IsGCZBlob(infile)) - { - PanicAlertT("\"%s\" is already compressed! Cannot compress it further.", infile_path.c_str()); - return false; - } - + std::unique_ptr infile = CreateDisc(infile_path); if (!infile) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(infile->IsSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -182,11 +179,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi } DiscScrubber disc_scrubber; - std::unique_ptr volume; if (sub_type == 1) { - volume = CreateDisc(infile_path); - if (!volume || !disc_scrubber.SetupScrub(volume.get(), block_size)) + if (!disc_scrubber.SetupScrub(infile.get())) { PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", infile_path.c_str()); @@ -206,7 +201,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi header.magic_cookie = GCZ_MAGIC; header.sub_type = sub_type; header.block_size = block_size; - header.data_size = infile.GetSize(); + header.data_size = infile->GetSize(); // round upwards! header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size); @@ -220,10 +215,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi outfile.Seek(sizeof(CompressedBlobHeader), SEEK_CUR); // seek past the offset and hash tables (we will write them at the end) outfile.Seek((sizeof(u64) + sizeof(u32)) * header.num_blocks, SEEK_CUR); - // seek to the start of the input file to make sure we get everything - infile.Seek(0, SEEK_SET); // Now we are ready to write compressed data! + u64 inpos = 0; u64 position = 0; int num_compressed = 0; int num_stored = 0; @@ -234,7 +228,6 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi { if (i % progress_monitor == 0) { - const u64 inpos = infile.Tell(); int ratio = 0; if (inpos != 0) ratio = (int)(100 * position / inpos); @@ -252,13 +245,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi offsets[i] = position; - size_t read_bytes; - if (scrubbing) - read_bytes = disc_scrubber.GetNextBlock(infile, in_buf.data()); - else - infile.ReadArray(in_buf.data(), header.block_size, &read_bytes); - if (read_bytes < header.block_size) - std::fill(in_buf.begin() + read_bytes, in_buf.begin() + header.block_size, 0); + const u64 bytes_to_read = scrubbing && disc_scrubber.CanBlockBeScrubbed(inpos) ? + 0 : + std::min(block_size, header.data_size - inpos); + + success = infile->Read(inpos, bytes_to_read, in_buf.data(), PARTITION_NONE); + if (!success) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + break; + } + + std::fill(in_buf.begin() + bytes_to_read, in_buf.begin() + header.block_size, 0); int retval = deflateReset(&z); z.next_in = in_buf.data(); @@ -305,6 +303,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi break; } + inpos += block_size; position += write_size; hashes[i] = Common::HashAdler32(write_buf, write_size); @@ -337,27 +336,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi return success; } -bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback, void* arg) +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback, void* arg) { - std::unique_ptr reader; - { - File::IOFile infile(infile_path, "rb"); - if (!IsGCZBlob(infile)) - { - PanicAlertT("File not compressed"); - return false; - } - - reader = CompressedBlobReader::Create(std::move(infile), infile_path); - } - + std::unique_ptr reader = CreateBlobReader(infile_path); if (!reader) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(reader->IsDataSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -368,11 +358,20 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out return false; } - const CompressedBlobHeader& header = reader->GetHeader(); - static const size_t BUFFER_BLOCKS = 32; - size_t buffer_size = header.block_size * BUFFER_BLOCKS; + constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; + u64 buffer_size = reader->GetBlockSize(); + if (buffer_size == 0) + { + buffer_size = DESIRED_BUFFER_SIZE; + } + else + { + while (buffer_size < DESIRED_BUFFER_SIZE) + buffer_size *= 2; + } + std::vector buffer(buffer_size); - u32 num_buffers = (header.num_blocks + BUFFER_BLOCKS - 1) / BUFFER_BLOCKS; + const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; int progress_monitor = std::max(1, num_buffers / 100); bool success = true; @@ -389,8 +388,13 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out } } const u64 inpos = i * buffer_size; - const u64 sz = std::min(buffer_size, header.data_size - inpos); - reader->Read(inpos, sz, buffer.data()); + const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); + if (!reader->Read(inpos, sz, buffer.data())) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + success = false; + break; + } if (!outfile.WriteBytes(buffer.data(), sz)) { PanicAlertT("Failed to write the output file \"%s\".\n" @@ -407,10 +411,6 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out outfile.Close(); File::Delete(outfile_path); } - else - { - outfile.Resize(header.data_size); - } return success; } diff --git a/Source/Core/DiscIO/CompressedBlob.h b/Source/Core/DiscIO/CompressedBlob.h index 0e290542c6..8c63aa68d6 100644 --- a/Source/Core/DiscIO/CompressedBlob.h +++ b/Source/Core/DiscIO/CompressedBlob.h @@ -52,6 +52,7 @@ public: u64 GetRawSize() const override { return m_file_size; } u64 GetDataSize() const override { return m_header.data_size; } bool IsDataSizeAccurate() const override { return true; } + u64 GetBlockSize() const override { return m_header.block_size; } u64 GetBlockCompressedSize(u64 block_num) const; bool GetBlock(u64 block_num, u8* out_ptr) override; diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index 8a06d9266e..07d504d72a 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -14,6 +14,7 @@ #include #include "Common/Align.h" +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/Logging/Log.h" @@ -29,19 +30,11 @@ constexpr size_t CLUSTER_SIZE = 0x8000; DiscScrubber::DiscScrubber() = default; DiscScrubber::~DiscScrubber() = default; -bool DiscScrubber::SetupScrub(const Volume* disc, int block_size) +bool DiscScrubber::SetupScrub(const Volume* disc) { if (!disc) return false; m_disc = disc; - m_block_size = block_size; - - if (CLUSTER_SIZE % m_block_size != 0) - { - ERROR_LOG(DISCIO, "Block size %u is not a factor of 0x8000, scrubbing not possible", - m_block_size); - return false; - } m_file_size = m_disc->GetSize(); @@ -54,34 +47,10 @@ bool DiscScrubber::SetupScrub(const Volume* disc, int block_size) // Fill out table of free blocks const bool success = ParseDisc(); - m_block_count = 0; - m_is_scrubbing = success; return success; } -size_t DiscScrubber::GetNextBlock(File::IOFile& in, u8* buffer) -{ - const u64 current_offset = m_block_count * m_block_size; - - size_t read_bytes = 0; - if (CanBlockBeScrubbed(current_offset)) - { - DEBUG_LOG(DISCIO, "Freeing 0x%016" PRIx64, current_offset); - std::fill(buffer, buffer + m_block_size, 0x00); - in.Seek(m_block_size, SEEK_CUR); - read_bytes = m_block_size; - } - else - { - DEBUG_LOG(DISCIO, "Used 0x%016" PRIx64, current_offset); - in.ReadArray(buffer, m_block_size, &read_bytes); - } - - m_block_count++; - return read_bytes; -} - bool DiscScrubber::CanBlockBeScrubbed(u64 offset) const { return m_is_scrubbing && m_free_table[offset / CLUSTER_SIZE]; diff --git a/Source/Core/DiscIO/DiscScrubber.h b/Source/Core/DiscIO/DiscScrubber.h index 15e0cda7f7..b3e7e7a9a0 100644 --- a/Source/Core/DiscIO/DiscScrubber.h +++ b/Source/Core/DiscIO/DiscScrubber.h @@ -34,8 +34,9 @@ public: DiscScrubber(); ~DiscScrubber(); - bool SetupScrub(const Volume* disc, int block_size); - size_t GetNextBlock(File::IOFile& in, u8* buffer); + bool SetupScrub(const Volume* disc); + + // Returns true if the specified 32 KiB block only contains unused data bool CanBlockBeScrubbed(u64 offset) const; private: @@ -72,8 +73,6 @@ private: std::vector m_free_table; u64 m_file_size = 0; - u64 m_block_count = 0; - u32 m_block_size = 0; bool m_is_scrubbing = false; }; diff --git a/Source/Core/DiscIO/DriveBlob.h b/Source/Core/DiscIO/DriveBlob.h index fddfbee7a2..46c141b1d3 100644 --- a/Source/Core/DiscIO/DriveBlob.h +++ b/Source/Core/DiscIO/DriveBlob.h @@ -23,11 +23,15 @@ class DriveReader : public SectorReader public: static std::unique_ptr Create(const std::string& drive); ~DriveReader(); + BlobType GetBlobType() const override { return BlobType::DRIVE; } + u64 GetRawSize() const override { return m_size; } u64 GetDataSize() const override { return m_size; } bool IsDataSizeAccurate() const override { return true; } + u64 GetBlockSize() const override { return ECC_BLOCK_SIZE; } + private: DriveReader(const std::string& drive); bool GetBlock(u64 block_num, u8* out_ptr) override; @@ -41,6 +45,7 @@ private: File::IOFile m_file; bool IsOK() const { return m_file.IsOpen() && m_file.IsGood(); } #endif + static constexpr u64 ECC_BLOCK_SIZE = 0x8000; u64 m_size = 0; }; diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 949457cd84..556d40c793 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -1031,7 +1031,7 @@ void VolumeVerifier::SetUpHashing() else if (m_volume.GetVolumeType() == Platform::WiiDisc) { // Set up a DiscScrubber for checking whether blocks with errors are unused - m_scrubber.SetupScrub(&m_volume, VolumeWii::BLOCK_TOTAL_SIZE); + m_scrubber.SetupScrub(&m_volume); } std::sort(m_blocks.begin(), m_blocks.end(), diff --git a/Source/Core/DiscIO/WbfsBlob.h b/Source/Core/DiscIO/WbfsBlob.h index 36de02f62d..0283b1c98b 100644 --- a/Source/Core/DiscIO/WbfsBlob.h +++ b/Source/Core/DiscIO/WbfsBlob.h @@ -24,6 +24,7 @@ public: static std::unique_ptr Create(File::IOFile file, const std::string& path); BlobType GetBlobType() const override { return BlobType::WBFS; } + u64 GetRawSize() const override { return m_size; } // The WBFS format does not save the original file size. // This function returns a constant upper bound @@ -31,6 +32,8 @@ public: u64 GetDataSize() const override; bool IsDataSizeAccurate() const override { return false; } + u64 GetBlockSize() const override { return m_wbfs_sector_size; } + bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DolphinQt/GameList/GameList.cpp b/Source/Core/DolphinQt/GameList/GameList.cpp index a635bee291..e4bf1c47bf 100644 --- a/Source/Core/DolphinQt/GameList/GameList.cpp +++ b/Source/Core/DolphinQt/GameList/GameList.cpp @@ -595,8 +595,8 @@ void GameList::CompressISO(bool decompress) } good = std::async(std::launch::async, [&] { - const bool good = DiscIO::DecompressBlobToFile(original_path, dst_path.toStdString(), - &CompressCB, &progress_dialog); + const bool good = DiscIO::ConvertToPlain(original_path, dst_path.toStdString(), &CompressCB, + &progress_dialog); progress_dialog.Reset(); return good; }); @@ -612,9 +612,9 @@ void GameList::CompressISO(bool decompress) good = std::async(std::launch::async, [&] { const bool good = - DiscIO::CompressFileToBlob(original_path, dst_path.toStdString(), - file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, - 16384, &CompressCB, &progress_dialog); + DiscIO::ConvertToGCZ(original_path, dst_path.toStdString(), + file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, + &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; });