Filesystem: Read the entire FST in one go

Instead of using lots of small scattered reads to read the FST,
only one big read is used, which is more efficient.

This also means that the FST only allocates memory once and stores all
strings close to each other - good for the CPU cache. The file info
objects use pointers to this FST memory of containing data themselves.
Keeping around the big m_FileInfoVector containing objects with only
pointers is a bit unnecessary, but that will be fixed soon.

Source/Core/DiscIO/FileSystemGCWii.cpp

@@ -22,8 +22,8 @@
 
 namespace DiscIO
 {
-FileInfoGCWii::FileInfoGCWii(u64 name_offset, u64 offset, u64 file_size, std::string name)
+FileInfoGCWii::FileInfoGCWii(u8 offset_shift, const u8* fst_entry, const u8* name_table_start)
-    : m_NameOffset(name_offset), m_Offset(offset), m_FileSize(file_size), m_Name(name)
+    : m_offset_shift(offset_shift), m_fst_entry(fst_entry), m_name_table_start(name_table_start)
 {
 }
 
@@ -31,6 +31,34 @@ FileInfoGCWii::~FileInfoGCWii()
 {
 }
 
+u32 FileInfoGCWii::Get(EntryProperty entry_property) const
+{
+  return Common::swap32(m_fst_entry + sizeof(u32) * static_cast<int>(entry_property));
+}
+
+u32 FileInfoGCWii::GetSize() const
+{
+  return Get(EntryProperty::FILE_SIZE);
+}
+
+u64 FileInfoGCWii::GetOffset() const
+{
+  return static_cast<u64>(Get(EntryProperty::FILE_OFFSET)) << (IsDirectory() ? 0 : m_offset_shift);
+}
+
+bool FileInfoGCWii::IsDirectory() const
+{
+  return (Get(EntryProperty::NAME_OFFSET) & 0xFF000000) != 0;
+}
+
+std::string FileInfoGCWii::GetName() const
+{
+  // TODO: Should we really always use SHIFT-JIS?
+  // Some names in Pikmin (NTSC-U) don't make sense without it, but is it correct?
+  const u8* name = m_name_table_start + (Get(EntryProperty::NAME_OFFSET) & 0xFFFFFF);
+  return SHIFTJISToUTF8(reinterpret_cast<const char*>(name));
+}
+
 FileSystemGCWii::FileSystemGCWii(const Volume* _rVolume, const Partition& partition)
     : FileSystem(_rVolume, partition), m_Initialized(false), m_Valid(false), m_offset_shift(0)
 {
@@ -55,6 +83,9 @@ const FileInfo* FileSystemGCWii::FindFileInfo(const std::string& path)
   if (!m_Initialized)
     InitFileSystem();
 
+  if (m_FileInfoVector.empty())
+    return nullptr;
+
   return FindFileInfo(path, 0);
 }
 
@@ -209,7 +240,7 @@ u64 FileSystemGCWii::ReadFile(const FileInfo* file_info, u8* _pBuffer, u64 _MaxB
   u64 read_length = std::min(_MaxBufferSize, file_info->GetSize() - _OffsetInFile);
 
   DEBUG_LOG(DISCIO, "Reading %" PRIx64 " bytes at %" PRIx64 " from file %s. Offset: %" PRIx64
-                    " Size: %" PRIx64,
+                    " Size: %" PRIx32,
             read_length, _OffsetInFile, GetPath(file_info->GetOffset()).c_str(),
             file_info->GetOffset(), file_info->GetSize());
 
@@ -343,17 +374,6 @@ bool FileSystemGCWii::ExportDOL(const std::string& _rExportFolder) const
   return false;
 }
 
-std::string FileSystemGCWii::GetStringFromOffset(u64 _Offset) const
-{
-  std::string data(255, 0x00);
-  m_rVolume->Read(_Offset, data.size(), (u8*)&data[0], m_partition);
-  data.erase(std::find(data.begin(), data.end(), 0x00), data.end());
-
-  // TODO: Should we really always use SHIFT-JIS?
-  // It makes some filenames in Pikmin (NTSC-U) sane, but is it correct?
-  return SHIFTJISToUTF8(data);
-}
-
 bool FileSystemGCWii::DetectFileSystem()
 {
   if (m_rVolume->ReadSwapped<u32>(0x18, m_partition) == u32(0x5D1C9EA3))
@@ -374,52 +394,41 @@ void FileSystemGCWii::InitFileSystem()
 {
   m_Initialized = true;
 
-  // read the whole FST
   const std::optional<u32> fst_offset_unshifted = m_rVolume->ReadSwapped<u32>(0x424, m_partition);
-  if (!fst_offset_unshifted)
+  const std::optional<u32> fst_size_unshifted = m_rVolume->ReadSwapped<u32>(0x428, m_partition);
+  if (!fst_offset_unshifted || !fst_size_unshifted)
     return;
-  const u64 FSTOffset = static_cast<u64>(*fst_offset_unshifted) << m_offset_shift;
+  const u64 fst_offset = static_cast<u64>(*fst_offset_unshifted) << m_offset_shift;
-
+  const u64 fst_size = static_cast<u64>(*fst_size_unshifted) << m_offset_shift;
-  // read all fileinfos
+  if (fst_size < 0xC)
-  const std::optional<u32> root_name_offset = m_rVolume->ReadSwapped<u32>(FSTOffset, m_partition);
-  const std::optional<u32> root_offset = m_rVolume->ReadSwapped<u32>(FSTOffset + 0x4, m_partition);
-  const std::optional<u32> root_size = m_rVolume->ReadSwapped<u32>(FSTOffset + 0x8, m_partition);
-  if (!root_name_offset || !root_offset || !root_size)
-    return;
-  FileInfoGCWii root(*root_name_offset, static_cast<u64>(*root_offset) << m_offset_shift,
-                     *root_size, "");
-
-  if (!root.IsDirectory())
     return;
 
-  // 12 bytes (the size of a file entry) times 10 * 1024 * 1024 is 120 MiB,
+  // 128 MiB is more than the total amount of RAM in a Wii.
-  // more than total RAM in a Wii. No file system should use anywhere near that much.
+  // No file system should use anywhere near that much.
-  static const u32 ARBITRARY_FILE_SYSTEM_SIZE_LIMIT = 10 * 1024 * 1024;
+  static const u32 ARBITRARY_FILE_SYSTEM_SIZE_LIMIT = 128 * 1024 * 1024;
-  if (root.GetSize() > ARBITRARY_FILE_SYSTEM_SIZE_LIMIT)
+  if (fst_size > ARBITRARY_FILE_SYSTEM_SIZE_LIMIT)
   {
     // Without this check, Dolphin can crash by trying to allocate too much
-    // memory when loading the file systems of certain malformed disc images.
+    // memory when loading a disc image with an incorrect FST size.
 
     ERROR_LOG(DISCIO, "File system is abnormally large! Aborting loading");
     return;
   }
 
-  if (m_FileInfoVector.size())
+  // Read the whole FST
-    PanicAlert("Wtf?");
+  m_file_system_table.resize(fst_size);
-  u64 NameTableOffset = FSTOffset + (root.GetSize() * 0xC);
+  if (!m_rVolume->Read(fst_offset, fst_size, m_file_system_table.data(), m_partition))
+    return;
 
-  m_FileInfoVector.reserve((size_t)root.GetSize());
+  // Create all file info objects
-  for (u32 i = 0; i < root.GetSize(); i++)
+  u32 number_of_file_infos = Common::swap32(*((u32*)m_file_system_table.data() + 2));
-  {
+  const u8* fst_start = m_file_system_table.data();
-    const u64 read_offset = FSTOffset + (i * 0xC);
+  const u8* name_table_start = fst_start + (number_of_file_infos * 0xC);
-    const u32 name_offset = m_rVolume->ReadSwapped<u32>(read_offset, m_partition).value_or(0);
+  const u8* name_table_end = fst_start + fst_size;
-    const u32 offset = m_rVolume->ReadSwapped<u32>(read_offset + 0x4, m_partition).value_or(0);
+  if (name_table_end < name_table_start)
-    const u32 size = m_rVolume->ReadSwapped<u32>(read_offset + 0x8, m_partition).value_or(0);
+    return;
-    const std::string name = GetStringFromOffset(NameTableOffset + (name_offset & 0xFFFFFF));
+  for (u32 i = 0; i < number_of_file_infos; i++)
-    m_FileInfoVector.emplace_back(
+    m_FileInfoVector.emplace_back(m_offset_shift, fst_start + (i * 0xC), name_table_start);
-        name_offset, static_cast<u64>(offset) << (m_offset_shift * !(name_offset & 0xFF000000)),
-        size, name);
-  }
 }
 
 }  // namespace

Source/Core/DiscIO/FileSystemGCWii.h

@@ -20,20 +20,29 @@ struct Partition;
 class FileInfoGCWii : public FileInfo
 {
 public:
-  FileInfoGCWii(u64 name_offset, u64 offset, u64 file_size, std::string name);
+  // Does not take ownership of pointers
+  FileInfoGCWii(u8 offset_shift, const u8* fst_entry, const u8* name_table_start);
+
   ~FileInfoGCWii() override;
 
-  u64 GetOffset() const override { return m_Offset; }
+  u64 GetOffset() const override;
-  u64 GetSize() const override { return m_FileSize; }
+  u32 GetSize() const override;
-  bool IsDirectory() const override { return (m_NameOffset & 0xFF000000) != 0; }
+  bool IsDirectory() const override;
-  const std::string& GetName() const override { return m_Name; }
+  std::string GetName() const override;
-  // TODO: These shouldn't be public
-  std::string m_Name;
-  const u64 m_NameOffset = 0u;
 
 private:
-  const u64 m_Offset = 0u;
+  enum class EntryProperty
-  const u64 m_FileSize = 0u;
+  {
+    NAME_OFFSET = 0,
+    FILE_OFFSET = 1,
+    FILE_SIZE = 2
+  };
+
+  u32 Get(EntryProperty entry_property) const;
+
+  const u8 m_offset_shift;
+  const u8* const m_fst_entry;
+  const u8* const m_name_table_start;
 };
 
 class FileSystemGCWii : public FileSystem
@@ -60,9 +69,9 @@ private:
   bool m_Valid;
   u32 m_offset_shift;
   std::vector<FileInfoGCWii> m_FileInfoVector;
+  std::vector<u8> m_file_system_table;
 
   const FileInfo* FindFileInfo(const std::string& path, size_t search_start_offset) const;
-  std::string GetStringFromOffset(u64 _Offset) const;
   bool DetectFileSystem();
   void InitFileSystem();
 };

Source/Core/DiscIO/Filesystem.h

@@ -26,9 +26,9 @@ public:
   // Not guaranteed to return a meaningful value for directories
   virtual u64 GetOffset() const = 0;
   // Not guaranteed to return a meaningful value for directories
-  virtual u64 GetSize() const = 0;
+  virtual u32 GetSize() const = 0;
   virtual bool IsDirectory() const = 0;
-  virtual const std::string& GetName() const = 0;
+  virtual std::string GetName() const = 0;
 };
 
 class FileSystem