Merge pull request #5088 from zhaowenlan1779/layered-fs

core/file_sys: LayeredFS support

src/citra_qt/game_list.cpp

@@ -468,6 +468,8 @@ void GameList::AddGamePopup(QMenu& context_menu, const QString& path, u64 progra
     QAction* open_texture_dump_location = context_menu.addAction(tr("Open Texture Dump Location"));
     QAction* open_texture_load_location =
         context_menu.addAction(tr("Open Custom Texture Location"));
+    QAction* open_mods_location = context_menu.addAction(tr("Open Mods Location"));
+    QAction* dump_romfs = context_menu.addAction(tr("Dump RomFS"));
     QAction* navigate_to_gamedb_entry = context_menu.addAction(tr("Navigate to GameDB entry"));
 
     const bool is_application =
@@ -497,6 +499,8 @@ void GameList::AddGamePopup(QMenu& context_menu, const QString& path, u64 progra
 
     open_texture_dump_location->setVisible(is_application);
     open_texture_load_location->setVisible(is_application);
+    open_mods_location->setVisible(is_application);
+    dump_romfs->setVisible(is_application);
 
     navigate_to_gamedb_entry->setVisible(it != compatibility_list.end());
 
@@ -526,6 +530,15 @@ void GameList::AddGamePopup(QMenu& context_menu, const QString& path, u64 progra
             emit OpenFolderRequested(program_id, GameListOpenTarget::TEXTURE_LOAD);
         }
     });
+    connect(open_mods_location, &QAction::triggered, [this, program_id] {
+        if (FileUtil::CreateFullPath(fmt::format("{}mods/{:016X}/",
+                                                 FileUtil::GetUserPath(FileUtil::UserPath::LoadDir),
+                                                 program_id))) {
+            emit OpenFolderRequested(program_id, GameListOpenTarget::MODS);
+        }
+    });
+    connect(dump_romfs, &QAction::triggered,
+            [this, path, program_id] { emit DumpRomFSRequested(path, program_id); });
     connect(navigate_to_gamedb_entry, &QAction::triggered, [this, program_id]() {
         emit NavigateToGamedbEntryRequested(program_id, compatibility_list);
     });

src/citra_qt/game_list.h

@@ -35,7 +35,8 @@ enum class GameListOpenTarget {
     APPLICATION = 2,
     UPDATE_DATA = 3,
     TEXTURE_DUMP = 4,
-    TEXTURE_LOAD = 5
+    TEXTURE_LOAD = 5,
+    MODS = 6,
 };
 
 class GameList : public QWidget {
@@ -81,6 +82,7 @@ signals:
     void OpenFolderRequested(u64 program_id, GameListOpenTarget target);
     void NavigateToGamedbEntryRequested(u64 program_id,
                                         const CompatibilityList& compatibility_list);
+    void DumpRomFSRequested(QString game_path, u64 program_id);
     void OpenDirectory(const QString& directory);
     void AddDirectory();
     void ShowList(bool show);

src/citra_qt/main.cpp

@@ -568,6 +568,7 @@ void GMainWindow::ConnectWidgetEvents() {
     connect(game_list, &GameList::OpenFolderRequested, this, &GMainWindow::OnGameListOpenFolder);
     connect(game_list, &GameList::NavigateToGamedbEntryRequested, this,
             &GMainWindow::OnGameListNavigateToGamedbEntry);
+    connect(game_list, &GameList::DumpRomFSRequested, this, &GMainWindow::OnGameListDumpRomFS);
     connect(game_list, &GameList::AddDirectory, this, &GMainWindow::OnGameListAddDirectory);
     connect(game_list_placeholder, &GameListPlaceholder::AddDirectory, this,
             &GMainWindow::OnGameListAddDirectory);
@@ -1144,6 +1145,11 @@ void GMainWindow::OnGameListOpenFolder(u64 data_id, GameListOpenTarget target) {
         path = fmt::format("{}textures/{:016X}/",
                            FileUtil::GetUserPath(FileUtil::UserPath::LoadDir), data_id);
         break;
+    case GameListOpenTarget::MODS:
+        open_target = "Mods";
+        path = fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir),
+                           data_id);
+        break;
     default:
         LOG_ERROR(Frontend, "Unexpected target {}", static_cast<int>(target));
         return;
@@ -1175,6 +1181,46 @@ void GMainWindow::OnGameListNavigateToGamedbEntry(u64 program_id,
     QDesktopServices::openUrl(QUrl(QStringLiteral("https://citra-emu.org/game/") + directory));
 }
 
+void GMainWindow::OnGameListDumpRomFS(QString game_path, u64 program_id) {
+    auto* dialog = new QProgressDialog(tr("Dumping..."), tr("Cancel"), 0, 0, this);
+    dialog->setWindowModality(Qt::WindowModal);
+    dialog->setWindowFlags(dialog->windowFlags() &
+                           ~(Qt::WindowCloseButtonHint | Qt::WindowContextHelpButtonHint));
+    dialog->setCancelButton(nullptr);
+    dialog->setMinimumDuration(0);
+    dialog->setValue(0);
+
+    const auto base_path = fmt::format(
+        "{}romfs/{:016X}", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir), program_id);
+    const auto update_path =
+        fmt::format("{}romfs/{:016X}", FileUtil::GetUserPath(FileUtil::UserPath::DumpDir),
+                    program_id | 0x0004000e00000000);
+    using FutureWatcher = QFutureWatcher<std::pair<Loader::ResultStatus, Loader::ResultStatus>>;
+    auto* future_watcher = new FutureWatcher(this);
+    connect(future_watcher, &FutureWatcher::finished,
+            [this, program_id, dialog, base_path, update_path, future_watcher] {
+                dialog->hide();
+                const auto& [base, update] = future_watcher->result();
+                if (base != Loader::ResultStatus::Success) {
+                    QMessageBox::critical(
+                        this, tr("Citra"),
+                        tr("Could not dump base RomFS.\nRefer to the log for details."));
+                    return;
+                }
+                QDesktopServices::openUrl(QUrl::fromLocalFile(QString::fromStdString(base_path)));
+                if (update == Loader::ResultStatus::Success) {
+                    QDesktopServices::openUrl(
+                        QUrl::fromLocalFile(QString::fromStdString(update_path)));
+                }
+            });
+
+    auto future = QtConcurrent::run([game_path, base_path, update_path] {
+        std::unique_ptr<Loader::AppLoader> loader = Loader::GetLoader(game_path.toStdString());
+        return std::make_pair(loader->DumpRomFS(base_path), loader->DumpUpdateRomFS(update_path));
+    });
+    future_watcher->setFuture(future);
+}
+
 void GMainWindow::OnGameListOpenDirectory(const QString& directory) {
     QString path;
     if (directory == QStringLiteral("INSTALLED")) {

src/citra_qt/main.h

@@ -169,6 +169,7 @@ private slots:
     void OnGameListOpenFolder(u64 program_id, GameListOpenTarget target);
     void OnGameListNavigateToGamedbEntry(u64 program_id,
                                          const CompatibilityList& compatibility_list);
+    void OnGameListDumpRomFS(QString game_path, u64 program_id);
     void OnGameListOpenDirectory(const QString& directory);
     void OnGameListAddDirectory();
     void OnGameListShowList(bool show);

src/core/CMakeLists.txt

@@ -72,6 +72,8 @@ add_library(core STATIC
     file_sys/delay_generator.h
     file_sys/ivfc_archive.cpp
     file_sys/ivfc_archive.h
+    file_sys/layered_fs.cpp
+    file_sys/layered_fs.h
     file_sys/ncch_container.cpp
     file_sys/ncch_container.h
     file_sys/patch.cpp

src/core/core.cpp

@@ -433,6 +433,7 @@ void System::Shutdown() {
     perf_stats.reset();
     rpc_server.reset();
     cheat_engine.reset();
+    archive_manager.reset();
     service_manager.reset();
     dsp_core.reset();
     cpu_cores.clear();

src/core/file_sys/layered_fs.cpp

@@ -0,0 +1,604 @@
+// Copyright 2020 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <algorithm>
+#include <cstring>
+#include "common/alignment.h"
+#include "common/assert.h"
+#include "common/common_paths.h"
+#include "common/file_util.h"
+#include "common/string_util.h"
+#include "common/swap.h"
+#include "core/file_sys/layered_fs.h"
+#include "core/file_sys/patch.h"
+
+namespace FileSys {
+
+struct FileRelocationInfo {
+    int type;                      // 0 - none, 1 - replaced / created, 2 - patched, 3 - removed
+    u64 original_offset;           // Type 0. Offset is absolute
+    std::string replace_file_path; // Type 1
+    std::vector<u8> patched_file;  // Type 2
+    u64 size;                      // Relocated file size
+};
+struct LayeredFS::File {
+    std::string name;
+    std::string path;
+    FileRelocationInfo relocation{};
+    Directory* parent;
+};
+
+struct DirectoryMetadata {
+    u32_le parent_directory_offset;
+    u32_le next_sibling_offset;
+    u32_le first_child_directory_offset;
+    u32_le first_file_offset;
+    u32_le hash_bucket_next;
+    u32_le name_length;
+    // Followed by a name of name length (aligned up to 4)
+};
+static_assert(sizeof(DirectoryMetadata) == 0x18, "Size of DirectoryMetadata is not correct");
+
+struct FileMetadata {
+    u32_le parent_directory_offset;
+    u32_le next_sibling_offset;
+    u64_le file_data_offset;
+    u64_le file_data_length;
+    u32_le hash_bucket_next;
+    u32_le name_length;
+    // Followed by a name of name length (aligned up to 4)
+};
+static_assert(sizeof(FileMetadata) == 0x20, "Size of FileMetadata is not correct");
+
+LayeredFS::LayeredFS(std::shared_ptr<RomFSReader> romfs_, std::string patch_path_,
+                     std::string patch_ext_path_, bool load_relocations)
+    : romfs(std::move(romfs_)), patch_path(std::move(patch_path_)),
+      patch_ext_path(std::move(patch_ext_path_)) {
+
+    romfs->ReadFile(0, sizeof(header), reinterpret_cast<u8*>(&header));
+
+    ASSERT_MSG(header.header_length == sizeof(header), "Header size is incorrect");
+
+    // TODO: is root always the first directory in table?
+    root.parent = &root;
+    LoadDirectory(root, 0);
+
+    if (load_relocations) {
+        LoadRelocations();
+        LoadExtRelocations();
+    }
+
+    RebuildMetadata();
+}
+
+LayeredFS::~LayeredFS() = default;
+
+void LayeredFS::LoadDirectory(Directory& current, u32 offset) {
+    DirectoryMetadata metadata;
+    romfs->ReadFile(header.directory_metadata_table.offset + offset, sizeof(metadata),
+                    reinterpret_cast<u8*>(&metadata));
+
+    current.name = ReadName(header.directory_metadata_table.offset + offset + sizeof(metadata),
+                            metadata.name_length);
+    current.path = current.parent->path + current.name + DIR_SEP;
+    directory_path_map.emplace(current.path, &current);
+
+    if (metadata.first_file_offset != 0xFFFFFFFF) {
+        LoadFile(current, metadata.first_file_offset);
+    }
+
+    if (metadata.first_child_directory_offset != 0xFFFFFFFF) {
+        auto child = std::make_unique<Directory>();
+        auto& directory = *child;
+        directory.parent = &current;
+        current.directories.emplace_back(std::move(child));
+        LoadDirectory(directory, metadata.first_child_directory_offset);
+    }
+
+    if (metadata.next_sibling_offset != 0xFFFFFFFF) {
+        auto sibling = std::make_unique<Directory>();
+        auto& directory = *sibling;
+        directory.parent = current.parent;
+        current.parent->directories.emplace_back(std::move(sibling));
+        LoadDirectory(directory, metadata.next_sibling_offset);
+    }
+}
+
+void LayeredFS::LoadFile(Directory& parent, u32 offset) {
+    FileMetadata metadata;
+    romfs->ReadFile(header.file_metadata_table.offset + offset, sizeof(metadata),
+                    reinterpret_cast<u8*>(&metadata));
+
+    auto file = std::make_unique<File>();
+    file->name = ReadName(header.file_metadata_table.offset + offset + sizeof(metadata),
+                          metadata.name_length);
+    file->path = parent.path + file->name;
+    file->relocation.original_offset = header.file_data_offset + metadata.file_data_offset;
+    file->relocation.size = metadata.file_data_length;
+    file->parent = &parent;
+
+    file_path_map.emplace(file->path, file.get());
+    parent.files.emplace_back(std::move(file));
+
+    if (metadata.next_sibling_offset != 0xFFFFFFFF) {
+        LoadFile(parent, metadata.next_sibling_offset);
+    }
+}
+
+std::string LayeredFS::ReadName(u32 offset, u32 name_length) {
+    std::vector<u16_le> buffer(name_length / sizeof(u16_le));
+    romfs->ReadFile(offset, name_length, reinterpret_cast<u8*>(buffer.data()));
+
+    std::u16string name(buffer.size(), 0);
+    std::transform(buffer.begin(), buffer.end(), name.begin(), [](u16_le character) {
+        return static_cast<char16_t>(static_cast<u16>(character));
+    });
+    return Common::UTF16ToUTF8(name);
+}
+
+void LayeredFS::LoadRelocations() {
+    if (!FileUtil::Exists(patch_path)) {
+        return;
+    }
+
+    const FileUtil::DirectoryEntryCallable callback = [this,
+                                                       &callback](u64* /*num_entries_out*/,
+                                                                  const std::string& directory,
+                                                                  const std::string& virtual_name) {
+        auto* parent = directory_path_map.at(directory.substr(patch_path.size() - 1));
+
+        if (FileUtil::IsDirectory(directory + virtual_name + DIR_SEP)) {
+            const auto path = (directory + virtual_name + DIR_SEP).substr(patch_path.size() - 1);
+            if (!directory_path_map.count(path)) { // Add this directory
+                auto directory = std::make_unique<Directory>();
+                directory->name = virtual_name;
+                directory->path = path;
+                directory->parent = parent;
+                directory_path_map.emplace(path, directory.get());
+                parent->directories.emplace_back(std::move(directory));
+                LOG_INFO(Service_FS, "LayeredFS created directory {}", path);
+            }
+            return FileUtil::ForeachDirectoryEntry(nullptr, directory + virtual_name + DIR_SEP,
+                                                   callback);
+        }
+
+        const auto path = (directory + virtual_name).substr(patch_path.size() - 1);
+        if (!file_path_map.count(path)) { // Newly created file
+            auto file = std::make_unique<File>();
+            file->name = virtual_name;
+            file->path = path;
+            file->parent = parent;
+            file_path_map.emplace(path, file.get());
+            parent->files.emplace_back(std::move(file));
+            LOG_INFO(Service_FS, "LayeredFS created file {}", path);
+        }
+
+        auto* file = file_path_map.at(path);
+        file->relocation.type = 1;
+        file->relocation.replace_file_path = directory + virtual_name;
+        file->relocation.size = FileUtil::GetSize(directory + virtual_name);
+        LOG_INFO(Service_FS, "LayeredFS replacement file in use for {}", path);
+        return true;
+    };
+
+    FileUtil::ForeachDirectoryEntry(nullptr, patch_path, callback);
+}
+
+void LayeredFS::LoadExtRelocations() {
+    if (!FileUtil::Exists(patch_ext_path)) {
+        return;
+    }
+
+    if (patch_ext_path.back() == '/' || patch_ext_path.back() == '\\') {
+        // ScanDirectoryTree expects a path without trailing '/'
+        patch_ext_path.erase(patch_ext_path.size() - 1, 1);
+    }
+
+    FileUtil::FSTEntry result;
+    FileUtil::ScanDirectoryTree(patch_ext_path, result, 256);
+
+    for (const auto& entry : result.children) {
+        if (FileUtil::IsDirectory(entry.physicalName)) {
+            continue;
+        }
+
+        const auto path = entry.physicalName.substr(patch_ext_path.size());
+        if (path.size() >= 5 && path.substr(path.size() - 5) == ".stub") {
+            // Remove the corresponding file if exists
+            const auto file_path = path.substr(0, path.size() - 5);
+            if (file_path_map.count(file_path)) {
+                auto& file = *file_path_map[file_path];
+                file.relocation.type = 3;
+                file.relocation.size = 0;
+                file_path_map.erase(file_path);
+                LOG_INFO(Service_FS, "LayeredFS removed file {}", file_path);
+            } else {
+                LOG_WARNING(Service_FS, "LayeredFS file for stub {} not found", path);
+            }
+        } else if (path.size() >= 4) {
+            const auto extension = path.substr(path.size() - 4);
+            if (extension != ".ips" && extension != ".bps") {
+                LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path);
+            }
+
+            const auto file_path = path.substr(0, path.size() - 4);
+            if (!file_path_map.count(file_path)) {
+                LOG_WARNING(Service_FS, "LayeredFS original file for patch {} not found", path);
+                continue;
+            }
+
+            FileUtil::IOFile patch_file(entry.physicalName, "rb");
+            if (!patch_file) {
+                LOG_ERROR(Service_FS, "LayeredFS Could not open file {}", entry.physicalName);
+                continue;
+            }
+
+            const auto size = patch_file.GetSize();
+            std::vector<u8> patch(size);
+            if (patch_file.ReadBytes(patch.data(), size) != size) {
+                LOG_ERROR(Service_FS, "LayeredFS Could not read file {}", entry.physicalName);
+                continue;
+            }
+
+            auto& file = *file_path_map[file_path];
+            std::vector<u8> buffer(file.relocation.size); // Original size
+            romfs->ReadFile(file.relocation.original_offset, buffer.size(), buffer.data());
+
+            bool ret = false;
+            if (extension == ".ips") {
+                ret = Patch::ApplyIpsPatch(patch, buffer);
+            } else {
+                ret = Patch::ApplyBpsPatch(patch, buffer);
+            }
+
+            if (ret) {
+                LOG_INFO(Service_FS, "LayeredFS patched file {}", file_path);
+
+                file.relocation.type = 2;
+                file.relocation.size = buffer.size();
+                file.relocation.patched_file = std::move(buffer);
+            } else {
+                LOG_ERROR(Service_FS, "LayeredFS failed to patch file {}", file_path);
+            }
+        } else {
+            LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path);
+        }
+    }
+}
+
+std::size_t GetNameSize(const std::string& name) {
+    std::u16string u16name = Common::UTF8ToUTF16(name);
+    return Common::AlignUp(u16name.size() * 2, 4);
+}
+
+void LayeredFS::PrepareBuildDirectory(Directory& current) {
+    directory_metadata_offset_map.emplace(&current, current_directory_offset);
+    directory_list.emplace_back(&current);
+    current_directory_offset += sizeof(DirectoryMetadata) + GetNameSize(current.name);
+}
+
+void LayeredFS::PrepareBuildFile(File& current) {
+    if (current.relocation.type == 3) { // Deleted files are not counted
+        return;
+    }
+    file_metadata_offset_map.emplace(&current, current_file_offset);
+    file_list.emplace_back(&current);
+    current_file_offset += sizeof(FileMetadata) + GetNameSize(current.name);
+}
+
+void LayeredFS::PrepareBuild(Directory& current) {
+    for (const auto& child : current.files) {
+        PrepareBuildFile(*child);
+    }
+
+    for (const auto& child : current.directories) {
+        PrepareBuildDirectory(*child);
+    }
+
+    for (const auto& child : current.directories) {
+        PrepareBuild(*child);
+    }
+}
+
+// Implementation from 3dbrew
+u32 CalcHash(const std::string& name, u32 parent_offset) {
+    u32 hash = parent_offset ^ 123456789;
+    std::u16string u16name = Common::UTF8ToUTF16(name);
+    for (char16_t c : u16name) {
+        hash = (hash >> 5) | (hash << 27);
+        hash ^= static_cast<u16>(c);
+    }
+    return hash;
+}
+
+std::size_t WriteName(u8* dest, std::u16string name) {
+    const auto buffer_size = Common::AlignUp(name.size() * 2, 4);
+    std::vector<u16_le> buffer(buffer_size / 2);
+    std::transform(name.begin(), name.end(), buffer.begin(), [](char16_t character) {
+        return static_cast<u16_le>(static_cast<u16>(character));
+    });
+    std::memcpy(dest, buffer.data(), buffer_size);
+
+    return buffer_size;
+}
+
+void LayeredFS::BuildDirectories() {
+    directory_metadata_table.resize(current_directory_offset, 0xFF);
+
+    std::size_t written = 0;
+    for (const auto& directory : directory_list) {
+        DirectoryMetadata metadata;
+        std::memset(&metadata, 0xFF, sizeof(metadata));
+        metadata.parent_directory_offset = directory_metadata_offset_map.at(directory->parent);
+
+        if (directory->parent != directory) {
+            bool flag = false;
+            for (const auto& sibling : directory->parent->directories) {
+                if (flag) {
+                    metadata.next_sibling_offset = directory_metadata_offset_map.at(sibling.get());
+                    break;
+                } else if (sibling.get() == directory) {
+                    flag = true;
+                }
+            }
+        }
+
+        if (!directory->directories.empty()) {
+            metadata.first_child_directory_offset =
+                directory_metadata_offset_map.at(directory->directories.front().get());
+        }
+
+        if (!directory->files.empty()) {
+            metadata.first_file_offset =
+                file_metadata_offset_map.at(directory->files.front().get());
+        }
+
+        const auto bucket = CalcHash(directory->name, metadata.parent_directory_offset) %
+                            directory_hash_table.size();
+        metadata.hash_bucket_next = directory_hash_table[bucket];
+        directory_hash_table[bucket] = directory_metadata_offset_map.at(directory);
+
+        // Write metadata and name
+        std::u16string u16name = Common::UTF8ToUTF16(directory->name);
+        metadata.name_length = u16name.size() * 2;
+
+        std::memcpy(directory_metadata_table.data() + written, &metadata, sizeof(metadata));
+        written += sizeof(metadata);
+
+        written += WriteName(directory_metadata_table.data() + written, u16name);
+    }
+
+    ASSERT_MSG(written == directory_metadata_table.size(),
+               "Calculated size for directory metadata table is wrong");
+}
+
+void LayeredFS::BuildFiles() {
+    file_metadata_table.resize(current_file_offset, 0xFF);
+
+    std::size_t written = 0;
+    for (const auto& file : file_list) {
+        FileMetadata metadata;
+        std::memset(&metadata, 0xFF, sizeof(metadata));
+
+        metadata.parent_directory_offset = directory_metadata_offset_map.at(file->parent);
+
+        bool flag = false;
+        for (const auto& sibling : file->parent->files) {
+            if (sibling->relocation.type == 3) { // removed file
+                continue;
+            }
+            if (flag) {
+                metadata.next_sibling_offset = file_metadata_offset_map.at(sibling.get());
+                break;
+            } else if (sibling.get() == file) {
+                flag = true;
+            }
+        }
+
+        metadata.file_data_offset = current_data_offset;
+        metadata.file_data_length = file->relocation.size;
+        current_data_offset += Common::AlignUp(metadata.file_data_length, 16);
+        if (metadata.file_data_length != 0) {
+            data_offset_map.emplace(metadata.file_data_offset, file);
+        }
+
+        const auto bucket =
+            CalcHash(file->name, metadata.parent_directory_offset) % file_hash_table.size();
+        metadata.hash_bucket_next = file_hash_table[bucket];
+        file_hash_table[bucket] = file_metadata_offset_map.at(file);
+
+        // Write metadata and name
+        std::u16string u16name = Common::UTF8ToUTF16(file->name);
+        metadata.name_length = u16name.size() * 2;
+
+        std::memcpy(file_metadata_table.data() + written, &metadata, sizeof(metadata));
+        written += sizeof(metadata);
+
+        written += WriteName(file_metadata_table.data() + written, u16name);
+    }
+
+    ASSERT_MSG(written == file_metadata_table.size(),
+               "Calculated size for file metadata table is wrong");
+}
+
+// Implementation from 3dbrew
+std::size_t GetHashTableSize(std::size_t entry_count) {
+    if (entry_count < 3) {
+        return 3;
+    } else if (entry_count < 19) {
+        return entry_count | 1;
+    } else {
+        std::size_t count = entry_count;
+        while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 ||
+               count % 11 == 0 || count % 13 == 0 || count % 17 == 0) {
+            count++;
+        }
+        return count;
+    }
+}
+
+void LayeredFS::RebuildMetadata() {
+    PrepareBuildDirectory(root);
+    PrepareBuild(root);
+
+    directory_hash_table.resize(GetHashTableSize(directory_list.size()), 0xFFFFFFFF);
+    file_hash_table.resize(GetHashTableSize(file_list.size()), 0xFFFFFFFF);
+
+    BuildDirectories();
+    BuildFiles();
+
+    // Create header
+    RomFSHeader header;
+    header.header_length = sizeof(header);
+    header.directory_hash_table = {
+        /*offset*/ sizeof(header),
+        /*length*/ static_cast<u32_le>(directory_hash_table.size() * sizeof(u32_le))};
+    header.directory_metadata_table = {
+        /*offset*/
+        header.directory_hash_table.offset + header.directory_hash_table.length,
+        /*length*/ static_cast<u32_le>(directory_metadata_table.size())};
+    header.file_hash_table = {
+        /*offset*/
+        header.directory_metadata_table.offset + header.directory_metadata_table.length,
+        /*length*/ static_cast<u32_le>(file_hash_table.size() * sizeof(u32_le))};
+    header.file_metadata_table = {/*offset*/ header.file_hash_table.offset +
+                                      header.file_hash_table.length,
+                                  /*length*/ static_cast<u32_le>(file_metadata_table.size())};
+    header.file_data_offset =
+        Common::AlignUp(header.file_metadata_table.offset + header.file_metadata_table.length, 16);
+
+    // Write hash table and metadata table
+    metadata.resize(header.file_data_offset);
+    std::memcpy(metadata.data(), &header, header.header_length);
+    std::memcpy(metadata.data() + header.directory_hash_table.offset, directory_hash_table.data(),
+                header.directory_hash_table.length);
+    std::memcpy(metadata.data() + header.directory_metadata_table.offset,
+                directory_metadata_table.data(), header.directory_metadata_table.length);
+    std::memcpy(metadata.data() + header.file_hash_table.offset, file_hash_table.data(),
+                header.file_hash_table.length);
+    std::memcpy(metadata.data() + header.file_metadata_table.offset, file_metadata_table.data(),
+                header.file_metadata_table.length);
+}
+
+std::size_t LayeredFS::GetSize() const {
+    return metadata.size() + current_data_offset;
+}
+
+std::size_t LayeredFS::ReadFile(std::size_t offset, std::size_t length, u8* buffer) {
+    ASSERT_MSG(offset + length <= GetSize(), "Out of bound");
+
+    std::size_t read_size = 0;
+    if (offset < metadata.size()) {
+        // First read the metadata
+        const auto to_read = std::min(metadata.size() - offset, length);
+        std::memcpy(buffer, metadata.data() + offset, to_read);
+        read_size += to_read;
+        offset = 0;
+    } else {
+        offset -= metadata.size();
+    }
+
+    // Read files
+    auto current = (--data_offset_map.upper_bound(offset));
+    while (read_size < length) {
+        const auto relative_offset = offset - current->first;
+        std::size_t to_read{};
+        if (current->second->relocation.size > relative_offset) {
+            to_read = std::min<std::size_t>(current->second->relocation.size - relative_offset,
+                                            length - read_size);
+        }
+        const auto alignment =
+            std::min<std::size_t>(Common::AlignUp(current->second->relocation.size, 16) -
+                                      relative_offset,
+                                  length - read_size) -
+            to_read;
+
+        // Read the file in different ways depending on relocation type
+        auto& relocation = current->second->relocation;
+        if (relocation.type == 0) { // none
+            romfs->ReadFile(relocation.original_offset + relative_offset, to_read,
+                            buffer + read_size);
+        } else if (relocation.type == 1) { // replace
+            FileUtil::IOFile replace_file(relocation.replace_file_path, "rb");
+            if (replace_file) {
+                replace_file.Seek(relative_offset, SEEK_SET);
+                replace_file.ReadBytes(buffer + read_size, to_read);
+            } else {
+                LOG_ERROR(Service_FS, "Could not open replacement file for {}",
+                          current->second->path);
+            }
+        } else if (relocation.type == 2) { // patch
+            std::memcpy(buffer + read_size, relocation.patched_file.data() + relative_offset,
+                        to_read);
+        } else {
+            UNREACHABLE();
+        }
+
+        std::memset(buffer + read_size + to_read, 0, alignment);
+
+        read_size += to_read + alignment;
+        offset += to_read + alignment;
+        current++;
+    }
+
+    return read_size;
+}
+
+bool LayeredFS::ExtractDirectory(Directory& current, const std::string& target_path) {
+    if (!FileUtil::CreateFullPath(target_path + current.path)) {
+        LOG_ERROR(Service_FS, "Could not create path {}", target_path + current.path);
+        return false;
+    }
+
+    constexpr std::size_t BufferSize = 0x10000;
+    std::array<u8, BufferSize> buffer;
+    for (const auto& file : current.files) {
+        // Extract file
+        const auto path = target_path + file->path;
+        LOG_INFO(Service_FS, "Extracting {} to {}", file->path, path);
+
+        FileUtil::IOFile target_file(path, "wb");
+        if (!target_file) {
+            LOG_ERROR(Service_FS, "Could not open file {}", path);
+            return false;
+        }
+
+        std::size_t written = 0;
+        while (written < file->relocation.size) {
+            const auto to_read =
+                std::min<std::size_t>(buffer.size(), file->relocation.size - written);
+            if (romfs->ReadFile(file->relocation.original_offset + written, to_read,
+                                buffer.data()) != to_read) {
+                LOG_ERROR(Service_FS, "Could not read from RomFS");
+                return false;
+            }
+
+            if (target_file.WriteBytes(buffer.data(), to_read) != to_read) {
+                LOG_ERROR(Service_FS, "Could not write to file {}", path);
+                return false;
+            }
+
+            written += to_read;
+        }
+    }
+
+    for (const auto& directory : current.directories) {
+        if (!ExtractDirectory(*directory, target_path)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+bool LayeredFS::DumpRomFS(const std::string& target_path) {
+    std::string path = target_path;
+    if (path.back() == '/' || path.back() == '\\') {
+        path.erase(path.size() - 1, 1);
+    }
+
+    return ExtractDirectory(root, path);
+}
+
+} // namespace FileSys

src/core/file_sys/layered_fs.h

@@ -0,0 +1,123 @@
+// Copyright 2020 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "common/common_types.h"
+#include "common/swap.h"
+#include "core/file_sys/romfs_reader.h"
+
+namespace FileSys {
+
+struct RomFSHeader {
+    struct Descriptor {
+        u32_le offset;
+        u32_le length;
+    };
+    u32_le header_length;
+    Descriptor directory_hash_table;
+    Descriptor directory_metadata_table;
+    Descriptor file_hash_table;
+    Descriptor file_metadata_table;
+    u32_le file_data_offset;
+};
+static_assert(sizeof(RomFSHeader) == 0x28, "Size of RomFSHeader is not correct");
+
+/**
+ * LayeredFS implementation. This basically adds a layer to another RomFSReader.
+ *
+ * patch_path: Path for RomFS replacements. Files present in this path replace or create
+ * corresponding files in RomFS.
+ * patch_ext_path: Path for RomFS extensions. Files present in this path:
+ *  - When with an extension of ".stub", remove the corresponding file in the RomFS.
+ *  - When with an extension of ".ips" or ".bps", patch the file in the RomFS.
+ */
+class LayeredFS : public RomFSReader {
+public:
+    explicit LayeredFS(std::shared_ptr<RomFSReader> romfs, std::string patch_path,
+                       std::string patch_ext_path, bool load_relocations = true);
+    ~LayeredFS() override;
+
+    std::size_t GetSize() const override;
+    std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) override;
+
+    bool DumpRomFS(const std::string& target_path);
+
+private:
+    struct File;
+    struct Directory {
+        std::string name;
+        std::string path; // with trailing '/'
+        std::vector<std::unique_ptr<File>> files;
+        std::vector<std::unique_ptr<Directory>> directories;
+        Directory* parent;
+    };
+
+    std::string ReadName(u32 offset, u32 name_length);
+
+    // Loads the current directory, then its siblings, and then its children.
+    void LoadDirectory(Directory& current, u32 offset);
+
+    // Load the file at offset, and then its siblings.
+    void LoadFile(Directory& parent, u32 offset);
+
+    // Load replace/create relocations
+    void LoadRelocations();
+
+    // Load patch/remove relocations
+    void LoadExtRelocations();
+
+    // Calculate the offset of a single directory add it to the map and list of directories
+    void PrepareBuildDirectory(Directory& current);
+
+    // Calculate the offset of a single file add it to the map and list of files
+    void PrepareBuildFile(File& current);
+
+    // Recursively generate a sequence of files and directories and their offsets for all
+    // children of current. (The current directory itself is not handled.)
+    void PrepareBuild(Directory& current);
+
+    void BuildDirectories();
+    void BuildFiles();
+
+    // Recursively extract a directory and all its contents to target_path
+    // target_path should be without trailing '/'.
+    bool ExtractDirectory(Directory& current, const std::string& target_path);
+
+    void RebuildMetadata();
+
+    std::shared_ptr<RomFSReader> romfs;
+    std::string patch_path;
+    std::string patch_ext_path;
+
+    RomFSHeader header;
+    Directory root;
+    std::unordered_map<std::string, File*> file_path_map;
+    std::unordered_map<std::string, Directory*> directory_path_map;
+    std::map<u64, File*> data_offset_map; // assigned data offset -> file
+    std::vector<u8> metadata;             // Includes header, hash table and metadata
+
+    // Used for rebuilding header
+    std::vector<u32_le> directory_hash_table;
+    std::vector<u32_le> file_hash_table;
+
+    std::unordered_map<Directory*, u32>
+        directory_metadata_offset_map;        // directory -> metadata offset
+    std::vector<Directory*> directory_list;   // sequence of directories to be written to metadata
+    u64 current_directory_offset{};           // current directory metadata offset
+    std::vector<u8> directory_metadata_table; // rebuilt directory metadata table
+
+    std::unordered_map<File*, u32> file_metadata_offset_map; // file -> metadata offset
+    std::vector<File*> file_list;        // sequence of files to be written to metadata
+    u64 current_file_offset{};           // current file metadata offset
+    std::vector<u8> file_metadata_table; // rebuilt file metadata table
+    u64 current_data_offset{};           // current assigned data offset
+};
+
+} // namespace FileSys

src/core/file_sys/ncch_container.cpp

@@ -11,6 +11,7 @@
 #include "common/common_types.h"
 #include "common/logging/log.h"
 #include "core/core.h"
+#include "core/file_sys/layered_fs.h"
 #include "core/file_sys/ncch_container.h"
 #include "core/file_sys/patch.h"
 #include "core/file_sys/seed_db.h"
@@ -25,6 +26,14 @@ namespace FileSys {
 static const int kMaxSections = 8;   ///< Maximum number of sections (files) in an ExeFs
 static const int kBlockSize = 0x200; ///< Size of ExeFS blocks (in bytes)
 
+u64 GetModId(u64 program_id) {
+    constexpr u64 UPDATE_MASK = 0x0000000e'00000000;
+    if ((program_id & 0x000000ff'00000000) == UPDATE_MASK) { // Apply the mods to updates
+        return program_id & ~UPDATE_MASK;
+    }
+    return program_id;
+}
+
 /**
  * Get the decompressed size of an LZSS compressed ExeFS file
  * @param buffer Buffer of compressed file
@@ -303,8 +312,22 @@ Loader::ResultStatus NCCHContainer::Load() {
                 }
             }
 
-            FileUtil::IOFile exheader_override_file{filepath + ".exheader", "rb"};
-            const bool has_exheader_override = read_exheader(exheader_override_file);
+            const auto mods_path =
+                fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir),
+                            GetModId(ncch_header.program_id));
+            std::array<std::string, 2> exheader_override_paths{{
+                mods_path + "exheader.bin",
+                filepath + ".exheader",
+            }};
+
+            bool has_exheader_override = false;
+            for (const auto& path : exheader_override_paths) {
+                FileUtil::IOFile exheader_override_file{path, "rb"};
+                if (read_exheader(exheader_override_file)) {
+                    has_exheader_override = true;
+                    break;
+                }
+            }
             if (has_exheader_override) {
                 if (exheader_header.system_info.jump_id !=
                     exheader_header.arm11_system_local_caps.program_id) {
@@ -512,7 +535,13 @@ Loader::ResultStatus NCCHContainer::ApplyCodePatch(std::vector<u8>& code) const
         std::string path;
         bool (*patch_fn)(const std::vector<u8>& patch, std::vector<u8>& code);
     };
-    const std::array<PatchLocation, 2> patch_paths{{
+
+    const auto mods_path =
+        fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir),
+                    GetModId(ncch_header.program_id));
+    const std::array<PatchLocation, 4> patch_paths{{
+        {mods_path + "exefs/code.ips", Patch::ApplyIpsPatch},
+        {mods_path + "exefs/code.bps", Patch::ApplyBpsPatch},
         {filepath + ".exefsdir/code.ips", Patch::ApplyIpsPatch},
         {filepath + ".exefsdir/code.bps", Patch::ApplyBpsPatch},
     }};
@@ -551,8 +580,16 @@ Loader::ResultStatus NCCHContainer::LoadOverrideExeFSSection(const char* name,
     else
         return Loader::ResultStatus::Error;
 
-    std::string section_override = filepath + ".exefsdir/" + override_name;
-    FileUtil::IOFile section_file(section_override, "rb");
+    const auto mods_path =
+        fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir),
+                    GetModId(ncch_header.program_id));
+    std::array<std::string, 2> override_paths{{
+        mods_path + "exefs/" + override_name,
+        filepath + ".exefsdir/" + override_name,
+    }};
+
+    for (const auto& path : override_paths) {
+        FileUtil::IOFile section_file(path, "rb");
 
         if (section_file.IsOpen()) {
             auto section_size = section_file.GetSize();
@@ -560,14 +597,16 @@ Loader::ResultStatus NCCHContainer::LoadOverrideExeFSSection(const char* name,
 
             section_file.Seek(0, SEEK_SET);
             if (section_file.ReadBytes(&buffer[0], section_size) == section_size) {
-            LOG_WARNING(Service_FS, "File {} overriding built-in ExeFS file", section_override);
+                LOG_WARNING(Service_FS, "File {} overriding built-in ExeFS file", path);
                 return Loader::ResultStatus::Success;
             }
         }
+    }
     return Loader::ResultStatus::ErrorNotUsed;
 }
 
-Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr<RomFSReader>& romfs_file) {
+Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr<RomFSReader>& romfs_file,
+                                              bool use_layered_fs) {
     Loader::ResultStatus result = Load();
     if (result != Loader::ResultStatus::Success)
         return result;
@@ -597,14 +636,43 @@ Loader::ResultStatus NCCHContainer::ReadRomFS(std::shared_ptr<RomFSReader>& romf
     if (!romfs_file_inner.IsOpen())
         return Loader::ResultStatus::Error;
 
+    std::shared_ptr<RomFSReader> direct_romfs;
     if (is_encrypted) {
-        romfs_file = std::make_shared<RomFSReader>(std::move(romfs_file_inner), romfs_offset,
+        direct_romfs =
+            std::make_shared<DirectRomFSReader>(std::move(romfs_file_inner), romfs_offset,
                                                 romfs_size, secondary_key, romfs_ctr, 0x1000);
     } else {
-        romfs_file =
-            std::make_shared<RomFSReader>(std::move(romfs_file_inner), romfs_offset, romfs_size);
+        direct_romfs = std::make_shared<DirectRomFSReader>(std::move(romfs_file_inner),
+                                                           romfs_offset, romfs_size);
     }
 
+    const auto path =
+        fmt::format("{}mods/{:016X}/", FileUtil::GetUserPath(FileUtil::UserPath::LoadDir),
+                    GetModId(ncch_header.program_id));
+    if (use_layered_fs &&
+        (FileUtil::Exists(path + "romfs/") || FileUtil::Exists(path + "romfs_ext/"))) {
+
+        romfs_file = std::make_shared<LayeredFS>(std::move(direct_romfs), path + "romfs/",
+                                                 path + "romfs_ext/");
+    } else {
+        romfs_file = std::move(direct_romfs);
+    }
+
+    return Loader::ResultStatus::Success;
+}
+
+Loader::ResultStatus NCCHContainer::DumpRomFS(const std::string& target_path) {
+    std::shared_ptr<RomFSReader> direct_romfs;
+    Loader::ResultStatus result = ReadRomFS(direct_romfs, false);
+    if (result != Loader::ResultStatus::Success)
+        return result;
+
+    std::shared_ptr<LayeredFS> layered_fs =
+        std::make_shared<LayeredFS>(std::move(direct_romfs), "", "", false);
+
+    if (!layered_fs->DumpRomFS(target_path)) {
+        return Loader::ResultStatus::Error;
+    }
     return Loader::ResultStatus::Success;
 }
 
@@ -614,8 +682,9 @@ Loader::ResultStatus NCCHContainer::ReadOverrideRomFS(std::shared_ptr<RomFSReade
     if (FileUtil::Exists(split_filepath)) {
         FileUtil::IOFile romfs_file_inner(split_filepath, "rb");
         if (romfs_file_inner.IsOpen()) {
-            LOG_WARNING(Service_FS, "File {} overriding built-in RomFS", split_filepath);
-            romfs_file = std::make_shared<RomFSReader>(std::move(romfs_file_inner), 0,
+            LOG_WARNING(Service_FS, "File {} overriding built-in RomFS; LayeredFS not enabled",
+                        split_filepath);
+            romfs_file = std::make_shared<DirectRomFSReader>(std::move(romfs_file_inner), 0,
                                                              romfs_file_inner.GetSize());
             return Loader::ResultStatus::Success;
         }

src/core/file_sys/ncch_container.h

@@ -247,7 +247,15 @@ public:
      * @param size The size of the romfs
      * @return ResultStatus result of function
      */
-    Loader::ResultStatus ReadRomFS(std::shared_ptr<RomFSReader>& romfs_file);
+    Loader::ResultStatus ReadRomFS(std::shared_ptr<RomFSReader>& romfs_file,
+                                   bool use_layered_fs = true);
+
+    /**
+     * Dump the RomFS of the NCCH container to the user folder.
+     * @param target_path target path to dump to
+     * @return ResultStatus result of function.
+     */
+    Loader::ResultStatus DumpRomFS(const std::string& target_path);
 
     /**
      * Get the override RomFS of the NCCH container

src/core/file_sys/romfs_reader.cpp

@@ -5,7 +5,7 @@
 
 namespace FileSys {
 
-std::size_t RomFSReader::ReadFile(std::size_t offset, std::size_t length, u8* buffer) {
+std::size_t DirectRomFSReader::ReadFile(std::size_t offset, std::size_t length, u8* buffer) {
     if (length == 0)
         return 0; // Crypto++ does not like zero size buffer
     file.Seek(file_offset + offset, SEEK_SET);

src/core/file_sys/romfs_reader.h

@@ -6,23 +6,39 @@
 
 namespace FileSys {
 
+/**
+ * Interface for reading RomFS data.
+ */
 class RomFSReader {
 public:
-    RomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size)
+    virtual ~RomFSReader() = default;
+
+    virtual std::size_t GetSize() const = 0;
+    virtual std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) = 0;
+};
+
+/**
+ * A RomFS reader that directly reads the RomFS file.
+ */
+class DirectRomFSReader : public RomFSReader {
+public:
+    DirectRomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size)
         : is_encrypted(false), file(std::move(file)), file_offset(file_offset),
           data_size(data_size) {}
 
-    RomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size,
+    DirectRomFSReader(FileUtil::IOFile&& file, std::size_t file_offset, std::size_t data_size,
                       const std::array<u8, 16>& key, const std::array<u8, 16>& ctr,
                       std::size_t crypto_offset)
         : is_encrypted(true), file(std::move(file)), key(key), ctr(ctr), file_offset(file_offset),
           crypto_offset(crypto_offset), data_size(data_size) {}
 
-    std::size_t GetSize() const {
+    ~DirectRomFSReader() override = default;
+
+    std::size_t GetSize() const override {
         return data_size;
     }
 
-    std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer);
+    std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) override;
 
 private:
     bool is_encrypted;

src/core/loader/3dsx.cpp

@@ -309,7 +309,7 @@ ResultStatus AppLoader_THREEDSX::ReadRomFS(std::shared_ptr<FileSys::RomFSReader>
         if (!romfs_file_inner.IsOpen())
             return ResultStatus::Error;
 
-        romfs_file = std::make_shared<FileSys::RomFSReader>(std::move(romfs_file_inner),
+        romfs_file = std::make_shared<FileSys::DirectRomFSReader>(std::move(romfs_file_inner),
                                                                   romfs_offset, romfs_size);
 
         return ResultStatus::Success;

src/core/loader/loader.h

@@ -186,6 +186,15 @@ public:
         return ResultStatus::ErrorNotImplemented;
     }
 
+    /**
+     * Dump the RomFS of the applciation
+     * @param target_path The target path to dump to
+     * @return ResultStatus result of function
+     */
+    virtual ResultStatus DumpRomFS(const std::string& target_path) {
+        return ResultStatus::ErrorNotImplemented;
+    }
+
     /**
      * Get the update RomFS of the application
      * Since the RomFS can be huge, we return a file reference instead of copying to a buffer
@@ -196,6 +205,15 @@ public:
         return ResultStatus::ErrorNotImplemented;
     }
 
+    /**
+     * Dump the update RomFS of the applciation
+     * @param target_path The target path to dump to
+     * @return ResultStatus result of function
+     */
+    virtual ResultStatus DumpUpdateRomFS(const std::string& target_path) {
+        return ResultStatus::ErrorNotImplemented;
+    }
+
     /**
      * Get the title of the application
      * @param title Reference to store the application title into

src/core/loader/ncch.cpp

@@ -254,6 +254,18 @@ ResultStatus AppLoader_NCCH::ReadUpdateRomFS(std::shared_ptr<FileSys::RomFSReade
     return ResultStatus::Success;
 }
 
+ResultStatus AppLoader_NCCH::DumpRomFS(const std::string& target_path) {
+    return base_ncch.DumpRomFS(target_path);
+}
+
+ResultStatus AppLoader_NCCH::DumpUpdateRomFS(const std::string& target_path) {
+    u64 program_id;
+    ReadProgramId(program_id);
+    update_ncch.OpenFile(
+        Service::AM::GetTitleContentPath(Service::FS::MediaType::SDMC, program_id | UPDATE_MASK));
+    return update_ncch.DumpRomFS(target_path);
+}
+
 ResultStatus AppLoader_NCCH::ReadTitle(std::string& title) {
     std::vector<u8> data;
     Loader::SMDH smdh;

src/core/loader/ncch.h

@@ -59,6 +59,10 @@ public:
 
     ResultStatus ReadUpdateRomFS(std::shared_ptr<FileSys::RomFSReader>& romfs_file) override;
 
+    ResultStatus DumpRomFS(const std::string& target_path) override;
+
+    ResultStatus DumpUpdateRomFS(const std::string& target_path) override;
+
     ResultStatus ReadTitle(std::string& title) override;
 
 private: