Modding function hooks (#530)

This commit updates the runtime for function hooking and implements any recomp-side functionality needed for them to fully hook (namely ROM decompression). For more details, see the relevant N64Recomp and N64ModernRuntime PRs.

.github/workflows/validate.yml

@@ -9,7 +9,7 @@ on:
       N64RECOMP_COMMIT:
         type: string
         required: false
-        default: 'fc696046da3e703450559154d9370ca74c197f8b'
+        default: 'b18e0ca2dd359d62dcc019771f0ccc4a1302bd03'
       DXC_CHECKSUM:
         type: string
         required: false

CMakeLists.txt

@@ -150,6 +150,7 @@ set (SOURCES
     ${CMAKE_SOURCE_DIR}/src/game/debug.cpp
     ${CMAKE_SOURCE_DIR}/src/game/quicksaving.cpp
     ${CMAKE_SOURCE_DIR}/src/game/recomp_api.cpp
+    ${CMAKE_SOURCE_DIR}/src/game/rom_decompression.cpp
 
     ${CMAKE_SOURCE_DIR}/src/ui/ui_renderer.cpp
     ${CMAKE_SOURCE_DIR}/src/ui/ui_launcher.cpp

Zelda64RecompSyms

@@ -1 +1 @@
-Subproject commit 50d63debd58ce9f01957142ea91b67b6c7018c96
+Subproject commit a325f8fa5c48c925616bd43685ef14bbabc29537

include/zelda_game.h

@@ -1,9 +1,14 @@
 #ifndef __ZELDA_GAME_H__
 #define __ZELDA_GAME_H__
 
+#include <cstdint>
+#include <span>
+#include <vector>
+
 namespace zelda64 {
     void quicksave_save();
     void quicksave_load();
+    std::vector<uint8_t> decompress_mm(std::span<const uint8_t> compressed_rom);
 };
 
 #endif

lib/N64ModernRuntime

@@ -1 +1 @@
-Subproject commit 50029c70fdc5c90f08d6e1198df6462a54aa3f40
+Subproject commit cdfe41680904b122d7226d750d5df6f56f9086d6

patches/syms.ld

@@ -3,8 +3,8 @@ __start = 0x80000000;
 /* Dummy addresses that get recompiled into function calls */
 recomp_puts = 0x8F000000;
 recomp_exit = 0x8F000004;
-recomp_handle_quicksave_actions = 0x8F000008;
+/* recomp_handle_quicksave_actions = 0x8F000008;
-recomp_handle_quicksave_actions_main = 0x8F00000C;
+recomp_handle_quicksave_actions_main = 0x8F00000C; */
 osRecvMesg_recomp = 0x8F000010;
 osSendMesg_recomp = 0x8F000014;
 recomp_get_gyro_deltas = 0x8F000018;

src/game/rom_decompression.cpp

@@ -0,0 +1,168 @@
+#include <cassert>
+#include <cstring>
+#include <fstream>
+
+#include "zelda_game.h"
+
+void naive_copy(std::span<uint8_t> dst, std::span<const uint8_t> src) {
+    for (size_t i = 0; i < src.size(); i++) {
+        dst[i] = src[i];
+    }
+}
+
+void yaz0_decompress(std::span<const uint8_t> input, std::span<uint8_t> output) {
+    int32_t layoutBitIndex;
+    uint8_t layoutBits;
+
+    size_t input_pos = 0;
+    size_t output_pos = 0;
+
+    size_t input_size = input.size();
+    size_t output_size = output.size();
+
+    while (input_pos < input_size) {
+        int32_t layoutBitIndex = 0;
+        uint8_t layoutBits = input[input_pos++];
+
+        while (layoutBitIndex < 8 && input_pos < input_size && output_pos < output_size) {
+            if (layoutBits & 0x80) {
+                output[output_pos++] = input[input_pos++];
+            } else {
+                int32_t firstByte = input[input_pos++];
+                int32_t secondByte = input[input_pos++];
+                uint32_t bytes = firstByte << 8 | secondByte;
+                uint32_t offset = (bytes & 0x0FFF) + 1;
+                uint32_t length;
+
+                // Check how the group length is encoded
+                if ((firstByte & 0xF0) == 0) {
+                    // 3 byte encoding, 0RRRNN
+                    int32_t thirdByte = input[input_pos++];
+                    length = thirdByte + 0x12;
+                } else {
+                    // 2 byte encoding, NRRR
+                    length = ((bytes & 0xF000) >> 12) + 2;
+                }
+
+                naive_copy(output.subspan(output_pos, length), output.subspan(output_pos - offset, length));
+                output_pos += length;
+            }
+
+            layoutBitIndex++;
+            layoutBits <<= 1;
+        }
+    }
+}
+
+#ifdef _MSC_VER
+inline uint32_t byteswap(uint32_t val) {
+    return _byteswap_ulong(val);
+}
+#else
+constexpr uint32_t byteswap(uint32_t val) {
+    return __builtin_bswap32(val);
+}
+#endif
+
+// Produces a decompressed MM rom. This is only needed because the game has compressed code.
+// For other recomps using this repo as an example, you can omit the decompression routine and
+// set the corresponding fields in the GameEntry if the game doesn't have compressed code,
+// even if it does have compressed data.
+std::vector<uint8_t> zelda64::decompress_mm(std::span<const uint8_t> compressed_rom) {
+    // Sanity check the rom size and header. These should already be correct from the runtime's check,
+    // but it should prevent this file from accidentally being copied to another recomp.
+    if (compressed_rom.size() != 0x2000000) {
+        assert(false);
+        return {};
+    }
+
+    if (compressed_rom[0x3B] != 'N' || compressed_rom[0x3C] != 'Z' || compressed_rom[0x3D] != 'S' || compressed_rom[0x3E] != 'E') {
+        assert(false);
+        return {};
+    }
+
+    struct DmaDataEntry {
+        uint32_t vrom_start;
+        uint32_t vrom_end;
+        uint32_t rom_start;
+        uint32_t rom_end;
+
+        void bswap() {
+            vrom_start = byteswap(vrom_start);
+            vrom_end = byteswap(vrom_end);
+            rom_start = byteswap(rom_start);
+            rom_end = byteswap(rom_end);
+        }
+    };
+
+    DmaDataEntry cur_entry{};
+    size_t cur_entry_index = 0;
+
+    constexpr size_t dma_data_rom_addr = 0x1A500;
+
+    std::vector<uint8_t> ret{};
+    ret.resize(0x2F00000);
+
+    size_t content_end = 0;
+
+    do {
+        // Read the entry from the compressed rom.
+        size_t cur_entry_rom_address = dma_data_rom_addr + (cur_entry_index++) * sizeof(DmaDataEntry);
+        memcpy(&cur_entry, compressed_rom.data() + cur_entry_rom_address, sizeof(DmaDataEntry));
+        // Swap the entry to native endianness after reading from the big endian data.
+        cur_entry.bswap();
+
+        // Rom end being 0 means the data is already uncompressed, so copy it as-is to vrom start.
+        size_t entry_decompressed_size = cur_entry.vrom_end - cur_entry.vrom_start;
+        if (cur_entry.rom_end == 0) {
+            memcpy(ret.data() + cur_entry.vrom_start, compressed_rom.data() + cur_entry.rom_start, entry_decompressed_size);
+
+            // Edit the entry to account for it being in a new location now.
+            cur_entry.rom_start = cur_entry.vrom_start;
+        }
+        // Otherwise, decompress the input data into the output data.
+        else {
+            if (cur_entry.rom_end != cur_entry.rom_start) {
+                // Validate the presence of the yaz0 header.
+                if (compressed_rom[cur_entry.rom_start + 0] != 'Y' ||
+                    compressed_rom[cur_entry.rom_start + 1] != 'a' ||
+                    compressed_rom[cur_entry.rom_start + 2] != 'z' ||
+                    compressed_rom[cur_entry.rom_start + 3] != '0')
+                {
+                    assert(false);
+                    return {};
+                }
+                // Skip the yaz0 header.
+                size_t compressed_data_rom_start = cur_entry.rom_start + 0x10;
+                size_t entry_compressed_size = cur_entry.rom_end - compressed_data_rom_start;
+
+                std::span input_span = std::span{ compressed_rom }.subspan(compressed_data_rom_start, entry_compressed_size);
+                std::span output_span = std::span{ ret }.subspan(cur_entry.vrom_start, entry_decompressed_size);
+                yaz0_decompress(input_span, output_span);
+
+                // Edit the entry to account for it being decompressed now.
+                cur_entry.rom_start = cur_entry.vrom_start;
+                cur_entry.rom_end = 0;
+            }
+        }
+
+        if (entry_decompressed_size != 0) {
+            if (cur_entry.vrom_end > content_end) {
+                content_end = cur_entry.vrom_end;
+            }
+        }
+
+        // Swap the entry back to big endian for writing.
+        cur_entry.bswap();
+        // Write the modified entry to the decompressed rom.
+        memcpy(ret.data() + cur_entry_rom_address, &cur_entry, sizeof(DmaDataEntry));
+    } while (cur_entry.vrom_end != 0);
+
+    // Align the start of padding to the closest 0x1000 (matches decomp rom decompression behavior).
+    content_end = (content_end + 0x1000 - 1) & -0x1000;
+
+    // Write 0xFF as the padding.
+    std::fill(ret.begin() + content_end, ret.end(), 0xFF);
+
+    return ret;
+}

src/main/main.cpp

@@ -25,6 +25,7 @@
 #include "zelda_config.h"
 #include "zelda_sound.h"
 #include "zelda_render.h"
+#include "zelda_game.h"
 #include "ovl_patches.hpp"
 #include "librecomp/game.hpp"
 #include "librecomp/mods.hpp"
@@ -330,7 +331,9 @@ std::vector<recomp::GameEntry> supported_games = {
         .game_id = u8"mm.n64.us.1.0",
         .mod_game_id = "mm",
         .save_type = recomp::SaveType::Flashram,
-        .is_enabled = true,
+        .is_enabled = false,
+        .decompression_routine = zelda64::decompress_mm,
+        .has_compressed_code = true,
         .entrypoint_address = get_entrypoint_address(),
         .entrypoint = recomp_entrypoint,
     },
@@ -538,6 +541,8 @@ void disable_texture_pack(recomp::mods::ModContext& context, const recomp::mods:
 }
 
 int main(int argc, char** argv) {
+    (void)argc;
+    (void)argv;
     recomp::Version project_version{};
     if (!recomp::Version::from_string(version_string, project_version)) {
         ultramodern::error_handling::message_box(("Invalid version string: " + version_string).c_str());

src/main/register_patches.cpp

@@ -9,4 +9,5 @@ void zelda64::register_patches() {
     recomp::overlays::register_patches(mm_patches_bin, sizeof(mm_patches_bin), section_table, ARRLEN(section_table));
     recomp::overlays::register_base_exports(export_table);
     recomp::overlays::register_base_events(event_names);
+    recomp::overlays::register_manual_patch_symbols(manual_patch_symbols);
 }