WUMS 0.3 support: Store function symbol data inside global struct. Optimize relocator to only read needed data from global struct

relocator/src/ModuleDataMinimal.h

@@ -0,0 +1,77 @@
+/****************************************************************************
+ * Copyright (C) 2018-2021 Maschell
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ ****************************************************************************/
+
+#include "../../source/module/RelocationData.h"
+#include "../../source/module/HookData.h"
+
+#pragma once
+
+class ModuleDataMinimal {
+public:
+    ModuleDataMinimal() = default;
+
+    ~ModuleDataMinimal() = default;
+
+    void setExportName(const std::string &name) {
+        this->export_name = name;
+    }
+
+    [[nodiscard]] std::string getExportName() const {
+        return this->export_name;
+    }
+
+    void addRelocationData(const RelocationData &relocation_data) {
+        relocation_data_list.push_back(relocation_data);
+    }
+
+    [[nodiscard]] const std::vector<RelocationData> &getRelocationDataList() const {
+        return relocation_data_list;
+    }
+
+    void addHookData(const HookData &data) {
+        hook_data_list.push_back(data);
+    }
+
+    [[nodiscard]] const std::vector<HookData> &getHookDataList() const {
+        return hook_data_list;
+    }
+
+    void setEntrypoint(uint32_t addr) {
+        this->entrypoint = addr;
+    }
+
+    [[nodiscard]] uint32_t getEntrypoint() const {
+        return entrypoint;
+    }
+
+    [[nodiscard]] bool isInitBeforeRelocationDoneHook() const {
+        return this->initBeforeRelocationDoneHook;
+    }
+
+    void setInitBeforeRelocationDoneHook(bool value) {
+        this->initBeforeRelocationDoneHook = value;
+    }
+
+    bool relocationsDone = false;
+private:
+    std::vector<RelocationData> relocation_data_list;
+    std::vector<HookData> hook_data_list;
+    std::string export_name;
+    uint32_t entrypoint = 0;
+    bool initBeforeRelocationDoneHook = false;
+
+};

relocator/src/ModuleDataPersistence.cpp

@@ -3,8 +3,8 @@
 #include <coreinit/cache.h>
 #include <wums.h>
 
-std::vector<ModuleData> ModuleDataPersistence::loadModuleData(module_information_t *moduleInformation) {
-    std::vector<ModuleData> result;
+std::vector<ModuleDataMinimal> ModuleDataPersistence::loadModuleData(module_information_t *moduleInformation) {
+    std::vector<ModuleDataMinimal> result;
     if (moduleInformation == nullptr) {
         DEBUG_FUNCTION_LINE("moduleInformation == NULL\n");
         return result;
@@ -22,25 +22,12 @@ std::vector<ModuleData> ModuleDataPersistence::loadModuleData(module_information
     for (int32_t i = 0; i < module_count; i++) {
         // Copy data from struct.
         module_information_single_t *module_data = &(moduleInformation->module_data[i]);
-        ModuleData moduleData;
+        ModuleDataMinimal moduleData;
 
-        moduleData.setBSSLocation(module_data->bssAddr, module_data->bssSize);
-        moduleData.setSBSSLocation(module_data->sbssAddr, module_data->sbssSize);
         moduleData.setEntrypoint(module_data->entrypoint);
-        moduleData.setStartAddress(module_data->startAddress);
-        moduleData.setEndAddress(module_data->endAddress);
-        moduleData.setSkipEntrypoint(module_data->skipEntrypoint);
         moduleData.setInitBeforeRelocationDoneHook(module_data->initBeforeRelocationDoneHook);
-
         moduleData.setExportName(module_data->module_export_name);
 
-        for (auto &export_entry: module_data->export_entries) {
-            if (export_entry.address == 0) {
-                continue;
-            }
-            moduleData.addExportData(ExportData(static_cast<wums_entry_type_t>(export_entry.type), export_entry.name, reinterpret_cast<const void *>(export_entry.address)));
-        }
-
         for (auto &hook_entry: module_data->hook_entries) {
             if (hook_entry.target == 0) {
                 continue;
@@ -76,6 +63,7 @@ std::vector<ModuleData> ModuleDataPersistence::loadModuleData(module_information
 
             moduleData.addRelocationData(reloc);
         }
+
         result.push_back(moduleData);
     }
     return result;

relocator/src/ModuleDataPersistence.h

@@ -1,9 +1,10 @@
 #pragma once
 
 #include <wums.h>
-#include "../../source/module/ModuleData.h"
+#include <vector>
+#include "ModuleDataMinimal.h"
 
 class ModuleDataPersistence {
 public:
-    static std::vector<ModuleData> loadModuleData(module_information_t *moduleInformation);
+    static std::vector<ModuleDataMinimal> loadModuleData(module_information_t *moduleInformation);
 };

relocator/src/entry.cpp

@@ -1,20 +1,19 @@
 #include <vector>
 #include <string>
-#include <cstring>
 #include <cstdint>
 #include <coreinit/dynload.h>
 #include <coreinit/cache.h>
 #include <map>
 #include <algorithm>
 #include <coreinit/memexpheap.h>
+#include "utils/logger.h"
+#include "utils/memory.h"
 #include "../../source/module/RelocationData.h"
-#include "../../source/module/ModuleData.h"
 #include "ModuleDataPersistence.h"
 #include "ElfUtils.h"
 #include "utils/dynamic.h"
 #include "globals.h"
 #include "hooks.h"
-#include "utils/memory.h"
 
 MEMHeapHandle gHeapHandle __attribute__((section(".data"))) = nullptr;
 uint8_t gFunctionsPatched __attribute__((section(".data"))) = 0;
@@ -22,7 +21,7 @@ uint8_t gInitCalled __attribute__((section(".data"))) = 0;
 
 extern "C" void socket_lib_init();
 
-std::vector<ModuleData> OrderModulesByDependencies(const std::vector<ModuleData> &loadedModules);
+std::vector<ModuleDataMinimal> OrderModulesByDependencies(const std::vector<ModuleDataMinimal> &loadedModules);
 
 extern "C" void doStart(int argc, char **argv);
 // We need to wrap it to make sure the main function is called AFTER our code.
@@ -32,7 +31,7 @@ extern "C" int _start(int argc, char **argv) {
 
     static uint8_t ucSetupRequired = 1;
     if (ucSetupRequired) {
-        gHeapHandle = MEMCreateExpHeapEx((void *) (MEMORY_REGION_USABLE_HEAP_START), MEMORY_REGION_USABLE_HEAP_END - MEMORY_REGION_USABLE_HEAP_START, 0);
+        gHeapHandle = MEMCreateExpHeapEx((void *) (MEMORY_REGION_USABLE_HEAP_START), MEMORY_REGION_USABLE_HEAP_END - MEMORY_REGION_USABLE_HEAP_START, 1);
         ucSetupRequired = 0;
     }
 
@@ -110,7 +109,7 @@ bool doRelocation(std::vector<RelocationData> &relocData, relocation_trampolin_e
     return true;
 }
 
-bool ResolveRelocations(std::vector<ModuleData> &loadedModules, bool skipMemoryMappingModule) {
+bool ResolveRelocations(std::vector<ModuleDataMinimal> &loadedModules, bool skipMemoryMappingModule) {
     bool wasSuccessful = true;
 
     for (auto &curModule: loadedModules) {
@@ -132,14 +131,6 @@ bool ResolveRelocations(std::vector<ModuleData> &loadedModules, bool skipMemoryM
             curModule.relocationsDone = true;
 
         }
-        if (curModule.getBSSAddr() != 0) {
-            // DEBUG_FUNCTION_LINE("memset .bss %08X (%d)\n", curModule.getBSSAddr(), curModule.getBSSSize());
-            // memset((void *) curModule.getBSSAddr(), 0, curModule.getBSSSize());
-        }
-        if (curModule.getSBSSAddr() != 0) {
-            // DEBUG_FUNCTION_LINE("memset .sbss %08X (%d)\n", curModule.getSBSSAddr(), curModule.getSBSSSize());
-            // memset((void *) curModule.getSBSSAddr(), 0, curModule.getSBSSSize());
-        }
     }
     DCFlushRange((void *) MEMORY_REGION_START, MEMORY_REGION_SIZE);
     ICInvalidateRange((void *) MEMORY_REGION_START, MEMORY_REGION_SIZE);
@@ -151,8 +142,8 @@ extern "C" void doStart(int argc, char **argv) {
         gFunctionsPatched = 1;
     }
     DEBUG_FUNCTION_LINE("Loading module data\n");
-    std::vector<ModuleData> loadedModulesUnordered = ModuleDataPersistence::loadModuleData(gModuleData);
-    std::vector<ModuleData> loadedModules = OrderModulesByDependencies(loadedModulesUnordered);
+    auto loadedModulesUnordered = ModuleDataPersistence::loadModuleData(gModuleData);
+    auto loadedModules = OrderModulesByDependencies(loadedModulesUnordered);
 
     bool applicationEndHookLoaded = false;
     for (auto &curModule: loadedModules) {
@@ -238,8 +229,8 @@ extern "C" void doStart(int argc, char **argv) {
     //CallHook(loadedModules, WUMS_HOOK_FINI_WUT);
 }
 
-std::vector<ModuleData> OrderModulesByDependencies(const std::vector<ModuleData> &loadedModules) {
-    std::vector<ModuleData> finalOrder;
+std::vector<ModuleDataMinimal> OrderModulesByDependencies(const std::vector<ModuleDataMinimal> &loadedModules) {
+    std::vector<ModuleDataMinimal> finalOrder;
     std::vector<std::string> loadedModulesExportNames;
     std::vector<uint32_t> loadedModulesEntrypoints;
 

relocator/src/hooks.cpp

@@ -20,14 +20,14 @@ static const char **hook_names = (const char *[]) {
         "WUMS_HOOK_RELOCATIONS_DONE",
         "WUMS_HOOK_APPLICATION_REQUESTS_EXI"};
 
-void CallHook(const std::vector<ModuleData> &modules, wums_hook_type_t type) {
+void CallHook(const std::vector<ModuleDataMinimal> &modules, wums_hook_type_t type) {
     DEBUG_FUNCTION_LINE_VERBOSE("Calling hook of type %s [%d] for all modules\n", hook_names[type], type);
     for (auto &curModule: modules) {
         CallHook(curModule, type);
     }
 }
 
-void CallHook(const ModuleData &module, wums_hook_type_t type) {
+void CallHook(const ModuleDataMinimal &module, wums_hook_type_t type) {
     if (!module.relocationsDone) {
         DEBUG_FUNCTION_LINE("Hook not called because the relocations failed\n");
         return;

relocator/src/hooks.h

@@ -3,8 +3,8 @@
 #include <wums.h>
 
 #include <vector>
-#include "../../source/module/ModuleData.h"
+#include "ModuleDataMinimal.h"
 
-void CallHook(const std::vector<ModuleData> &modules, wums_hook_type_t type);
+void CallHook(const std::vector<ModuleDataMinimal> &modules, wums_hook_type_t type);
 
-void CallHook(const ModuleData &module, wums_hook_type_t type);
+void CallHook(const ModuleDataMinimal &module, wums_hook_type_t type);

relocator/src/utils/memory.cpp

@@ -28,21 +28,7 @@ extern MEMHeapHandle gHeapHandle;
 void *MEMAllocSafe(uint32_t size, uint32_t align) {
     void *res = nullptr;
     MEMHeapHandle heapHandle = gHeapHandle;
-    MEMExpHeap *heap = (MEMExpHeap *) heapHandle;
-    OSUninterruptibleSpinLock_Acquire(&heap->header.lock);
     res = MEMAllocFromExpHeapEx(heapHandle, size, align);
-    auto cur = heap->usedList.head;
-    while (cur != nullptr) {
-        DCFlushRange(cur, sizeof(MEMExpHeapBlock));
-        cur = cur->next;
-    }
-    cur = heap->freeList.head;
-    while (cur != nullptr) {
-        DCFlushRange(cur, sizeof(MEMExpHeapBlock));
-        cur = cur->next;
-    }
-    OSUninterruptibleSpinLock_Release(&heap->header.lock);
-
     return res;
 }
 

source/module/FunctionSymbolData.h

@@ -0,0 +1,56 @@
+/****************************************************************************
+ * Copyright (C) 2021 Maschell
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ ****************************************************************************/
+
+#pragma once
+
+#include <string>
+
+class FunctionSymbolData {
+
+public:
+    FunctionSymbolData(const FunctionSymbolData &o2) = default;
+
+    FunctionSymbolData(const char* name, void *address, uint32_t size) :
+            name(name),
+            address(address),
+            size(size) {
+    }
+
+    virtual ~FunctionSymbolData() = default;
+
+    bool operator<(const FunctionSymbolData &rhs) const {
+        return (uint32_t) address < (uint32_t) rhs.address;  //assume that you compare the record based on a
+    }
+
+    [[nodiscard]] const char *getName() const {
+        return name;
+    }
+
+    [[nodiscard]] void *getAddress() const {
+        return address;
+    }
+
+    [[nodiscard]] uint32_t getSize() const {
+        return size;
+    }
+
+
+private:
+    const char* name;
+    void *address;
+    uint32_t size;
+};

source/module/ModuleData.h

@@ -20,10 +20,12 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <set>
 #include "RelocationData.h"
 #include "SectionInfo.h"
 #include "ExportData.h"
 #include "HookData.h"
+#include "FunctionSymbolData.h"
 
 class ModuleData {
 public:
@@ -81,6 +83,14 @@ public:
         section_info_list[sectionInfo.getName()] = sectionInfo;
     }
 
+    void addFunctionSymbolData(const FunctionSymbolData &symbol_data) {
+        symbol_data_list.insert(symbol_data);
+    }
+
+    [[nodiscard]] const std::set<FunctionSymbolData> &getFunctionSymbolDataList() const {
+        return symbol_data_list;
+    }
+
     [[nodiscard]] const std::map<std::string, SectionInfo> &getSectionInfoList() const {
         return section_info_list;
     }
@@ -151,6 +161,7 @@ private:
     std::vector<RelocationData> relocation_data_list;
     std::vector<ExportData> export_data_list;
     std::vector<HookData> hook_data_list;
+    std::set<FunctionSymbolData> symbol_data_list;
     std::map<std::string, SectionInfo> section_info_list;
 
     std::string export_name;

source/module/ModuleDataFactory.cpp

@@ -23,6 +23,7 @@
 #include "ModuleDataFactory.h"
 #include "utils/utils.h"
 #include "ElfUtils.h"
+#include "FunctionSymbolData.h"
 
 using namespace ELFIO;
 
@@ -98,7 +99,7 @@ ModuleDataFactory::load(const std::string &path, uint32_t *destination_address_p
                 memcpy((void *) destination, p, sectionSize);
             }
 
-            //nextAddress = ROUNDUP(destination + sectionSize,0x100);
+            //nextAddress = ROUNDUP(destination + sectionSize, 0x100);
             if (psec->get_name() == ".bss") {
                 moduleData.setBSSLocation(destination, sectionSize);
                 memset(reinterpret_cast<void *>(destination), 0, sectionSize);
@@ -200,7 +201,7 @@ ModuleDataFactory::load(const std::string &path, uint32_t *destination_address_p
                             moduleData.setInitBeforeRelocationDoneHook(false);
                         }
                     } else if (key == "wums") {
-                        if (value != "0.2") {
+                        if (value != "0.3") {
                             DEBUG_FUNCTION_LINE("Warning: Ignoring module - Unsupported WUMS version: %s.\n", value.c_str());
                             return std::nullopt;
                         }
@@ -211,6 +212,49 @@ ModuleDataFactory::load(const std::string &path, uint32_t *destination_address_p
         }
     }
 
+    char *strTable = (char *) endAddress;
+    uint32_t strOffset = 0;
+
+    // Get the symbol for functions.
+    Elf_Half n = reader.sections.size();
+    for (Elf_Half i = 0; i < n; ++i) {
+        section *sec = reader.sections[i];
+        if (SHT_SYMTAB == sec->get_type()) {
+            symbol_section_accessor symbols(reader, sec);
+            auto sym_no = (uint32_t) symbols.get_symbols_num();
+            if (sym_no > 0) {
+                for (Elf_Half j = 0; j < sym_no; ++j) {
+                    std::string name;
+                    Elf64_Addr value = 0;
+                    Elf_Xword size = 0;
+                    unsigned char bind = 0;
+                    unsigned char type = 0;
+                    Elf_Half section = 0;
+                    unsigned char other = 0;
+                    if (symbols.get_symbol(j, name, value, size, bind, type, section, other)) {
+                        if (type == STT_FUNC) { // We only care about functions.
+                            auto sectionVal = reader.sections[section];
+                            auto offsetVal = value - sectionVal->get_address();
+                            auto sectionOpt = moduleData.getSectionInfo(sectionVal->get_name());
+                            if (!sectionOpt.has_value()) {
+                                continue;
+                            }
+                            auto finalAddress = offsetVal + sectionOpt->getAddress();
+
+                            uint32_t stringSize = name.size() + 1;
+                            memcpy(strTable + strOffset, name.c_str(), stringSize);
+                            moduleData.addFunctionSymbolData(FunctionSymbolData(strTable + strOffset, (void *) finalAddress, (uint32_t) size));
+                            strOffset += stringSize;
+                            totalSize += stringSize;
+                            endAddress += stringSize;
+                        }
+                    }
+                }
+                break;
+            }
+        }
+    }
+
     DCFlushRange((void *) *destination_address_ptr, totalSize);
     ICInvalidateRange((void *) *destination_address_ptr, totalSize);
 

source/module/ModuleDataPersistence.cpp

@@ -2,8 +2,6 @@
 
 #include "ModuleDataPersistence.h"
 #include "DynamicLinkingHelper.h"
-#include "ModuleData.h"
-#include "RelocationData.h"
 
 bool ModuleDataPersistence::saveModuleData(module_information_t *moduleInformation, const ModuleData &module) {
     int32_t module_count = moduleInformation->number_used_modules;
@@ -15,7 +13,7 @@ bool ModuleDataPersistence::saveModuleData(module_information_t *moduleInformati
     // Copy data to global struct.
     module_information_single_t *module_data = &(moduleInformation->module_data[module_count]);
 
-    DEBUG_FUNCTION_LINE("Saving reloation data for module at %08X", module.getEntrypoint());
+    DEBUG_FUNCTION_LINE("Saving relocation data for module at %08X", module.getEntrypoint());
     // Relocation
     std::vector<RelocationData> relocationData = module.getRelocationDataList();
     for (auto const &reloc: relocationData) {
@@ -64,6 +62,30 @@ bool ModuleDataPersistence::saveModuleData(module_information_t *moduleInformati
 
     strncpy(module_data->module_export_name, module.getExportName().c_str(), MAXIMUM_EXPORT_MODULE_NAME_LENGTH);
 
+    uint32_t entryCount = module.getFunctionSymbolDataList().size();
+    if (entryCount > 0) {
+        auto ptr = &moduleInformation->function_symbols[moduleInformation->number_used_function_symbols];
+        module_data->function_symbol_entries = ptr;
+
+        uint32_t sym_offset = 0;
+        for (auto &curFuncSym: module.getFunctionSymbolDataList()) {
+            if (moduleInformation->number_used_function_symbols >= FUNCTION_SYMBOL_LIST_LENGTH) {
+                DEBUG_FUNCTION_LINE("Function symbol list is full");
+                break;
+            }
+            module_data->function_symbol_entries[sym_offset].address = curFuncSym.getAddress();
+            module_data->function_symbol_entries[sym_offset].name = (char *) curFuncSym.getName();
+            module_data->function_symbol_entries[sym_offset].size = curFuncSym.getSize();
+
+            sym_offset++;
+            moduleInformation->number_used_function_symbols++;
+        }
+        module_data->number_used_function_symbols = sym_offset;
+    } else {
+        module_data->function_symbol_entries = nullptr;
+        module_data->number_used_function_symbols = 0;
+    }
+
     module_data->bssAddr = module.getBSSAddr();
     module_data->bssSize = module.getBSSSize();
     module_data->sbssAddr = module.getSBSSAddr();
@@ -144,6 +166,13 @@ std::vector<ModuleData> ModuleDataPersistence::loadModuleData(module_information
 
             moduleData.addRelocationData(reloc);
         }
+
+        if (module_data->function_symbol_entries != nullptr && module_data->number_used_function_symbols > 0) {
+            for (uint32_t j = 0; j < module_data->number_used_function_symbols; j++) {
+                auto symbol = &module_data->function_symbol_entries[j];
+                moduleData.addFunctionSymbolData(FunctionSymbolData(symbol->name, symbol->address, symbol->size));
+            }
+        }
         result.push_back(moduleData);
     }
     return result;