Support loading files which don't fit into memory by saving them to the sd card

Dockerfile

@@ -1,8 +1,8 @@
-FROM ghcr.io/wiiu-env/devkitppc:20240505
+FROM ghcr.io/wiiu-env/devkitppc:20241128
 
 COPY --from=ghcr.io/wiiu-env/wiiupluginsystem:20240505 /artifacts $DEVKITPRO
 COPY --from=ghcr.io/wiiu-env/libwupsbackend:20240425 /artifacts $DEVKITPRO
 COPY --from=ghcr.io/wiiu-env/librpxloader:20240425 /artifacts $DEVKITPRO
-COPY --from=ghcr.io/wiiu-env/libnotifications:20240426 /artifacts $DEVKITPRO
+COPY --from=ghcr.io/wiiu-env/libnotifications:20241221 /artifacts $DEVKITPRO
 
 WORKDIR project

src/config.cpp

@@ -3,10 +3,12 @@
 #include "utils/TcpReceiver.h"
 #include "utils/logger.h"
 #include "utils/utils.h"
-#include <string>
+
 #include <wups/config/WUPSConfigItemBoolean.h>
 #include <wups/storage.h>
 
+#include <string>
+
 static void gServerEnabledChanged(ConfigItemBoolean *item, bool newValue) {
     if (std::string_view(WIILOAD_ENABLED_STRING) != item->identifier) {
         DEBUG_FUNCTION_LINE_WARN("Unexpected identifier in bool callback: %s", item->identifier);

src/globals.cpp

@@ -1,4 +1,5 @@
 #include "globals.h"
+#include "utils/TcpReceiver.h"
 
 bool gLibRPXLoaderInitDone                      = false;
 std::unique_ptr<TcpReceiver> gTcpReceiverThread = nullptr;

src/globals.h

@@ -1,7 +1,7 @@
-#include "utils/TcpReceiver.h"
-#include <cstdint>
 #include <memory>
 
+class TcpReceiver;
+
 extern bool gLibRPXLoaderInitDone;
 extern std::unique_ptr<TcpReceiver> gTcpReceiverThread;
 extern bool gWiiloadServerEnabled;

src/main.cpp

@@ -4,9 +4,12 @@
 #include "utils/TcpReceiver.h"
 #include "utils/logger.h"
 #include "utils/utils.h"
-#include <coreinit/debug.h>
+
 #include <notifications/notifications.h>
 #include <rpxloader/rpxloader.h>
+
+#include <coreinit/debug.h>
+
 #include <wups.h>
 #include <wups/config/WUPSConfigItemBoolean.h>
 #include <wups_backend/api.h>

src/utils/CThread.h

@@ -20,6 +20,7 @@
 #include <coreinit/systeminfo.h>
 #include <coreinit/thread.h>
 #include <malloc.h>
+#include <string>
 #include <unistd.h>
 
 class CThread {

src/utils/FSUtils.cpp

@@ -1,8 +1,13 @@
 #include "FSUtils.h"
 #include "logger.h"
+#include "utils.h"
+
 #include <cstdio>
 #include <cstring>
 #include <fcntl.h>
+#include <notifications/notification_defines.h>
+#include <notifications/notifications.h>
+#include <string>
 #include <unistd.h>
 
 bool FSUtils::CheckFile(const char *filepath) {
@@ -82,7 +87,7 @@ bool FSUtils::CreateSubfolder(const char *fullpath) {
     return true;
 }
 
-bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size) {
+bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size, NotificationModuleHandle notificationHandle) {
     int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY);
     if (fd < 0) {
         DEBUG_FUNCTION_LINE_ERR("Failed to open %s. %d", path, fd);
@@ -93,6 +98,10 @@ bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size) {
     int32_t curResult;
     int64_t totalSizeWritten = 0;
     while (sizeToWrite > 0) {
+        if (notificationHandle != 0) {
+            std::string progressStr = string_format("[Wiiload] Write data to file %.2f%%", static_cast<float>(totalSizeWritten) / static_cast<float>(size) * 100.0);
+            NotificationModule_UpdateDynamicNotificationText(notificationHandle, progressStr.c_str());
+        }
         curResult = write(fd, ptr, sizeToWrite);
         if (curResult < 0) {
             close(fd);
@@ -106,5 +115,11 @@ bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size) {
         sizeToWrite -= curResult;
     }
     close(fd);
+
+    if (notificationHandle != 0) {
+        std::string progressStr = string_format("[Wiiload] Save data to file %.2f%%", static_cast<float>(totalSizeWritten) / static_cast<float>(size) * 100.0);
+        NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Write data to file 100%");
+    }
+
     return totalSizeWritten == size;
 }

src/utils/FSUtils.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <notifications/notification_defines.h>
 #include <wut_types.h>
 
 class FSUtils {
@@ -8,5 +9,5 @@ public:
 
     static bool CheckFile(const char *filepath);
 
-    static bool saveBufferToFile(const char *path, void *buffer, uint32_t size);
+    static bool saveBufferToFile(const char *path, void *buffer, uint32_t size, NotificationModuleHandle notificationHandle);
 };

src/utils/ReadWriteStreamIF.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <string>
+
+#include <cstdint>
+
+class ReadWriteStreamIF {
+public:
+    ReadWriteStreamIF() = default;
+
+    virtual ~ReadWriteStreamIF() = default;
+    // Delete the copy constructor and copy assignment operator
+    ReadWriteStreamIF(const ReadWriteStreamIF &) = delete;
+    ReadWriteStreamIF &operator=(const ReadWriteStreamIF &) = delete;
+
+    virtual bool isOpen() = 0;
+
+    // must not return false if a path can be returned
+    virtual bool isFile()         = 0;
+    virtual std::string getPath() = 0;
+
+    // must not return false if a data/size can be returned
+    virtual bool isDataWrapper() = 0;
+    virtual void *getData()      = 0;
+    virtual size_t getSize()     = 0;
+
+    virtual int64_t write(const void *data, size_t size) = 0;
+
+    // Read data from the memory stream
+    virtual int64_t read(void *output, size_t size) = 0;
+
+    // Seek to a specific position
+    virtual off_t seek(int64_t position, int whence) = 0;
+
+    // Get the current position
+    [[nodiscard]] virtual off_t tell() const = 0;
+};

src/utils/TcpReceiver.cpp

@@ -1,29 +1,292 @@
 #include "TcpReceiver.h"
+
 #include "FSUtils.h"
-#include "globals.h"
-#include "utils/net.h"
-#include "utils/utils.h"
-#include <algorithm>
-#include <coreinit/cache.h>
-#include <coreinit/debug.h>
-#include <coreinit/dynload.h>
-#include <coreinit/title.h>
-#include <cstring>
-#include <nn/act.h>
+#include "ReadWriteStreamIF.h"
+#include "net.h"
+#include "utils.h"
+
 #include <notifications/notifications.h>
 #include <rpxloader/rpxloader.h>
-#include <sysapp/launch.h>
-#include <vector>
 #include <wups_backend/PluginUtils.h>
+#include <wups_backend/import_defines.h>
+
+#include <coreinit/cache.h>
+#include <coreinit/title.h>
+#include <nn/act.h>
+#include <sysapp/launch.h>
+
 #include <zlib.h>
 
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <cstdint>
+#include <cstring>
+#include <netinet/in.h>
+#include <sys/socket.h>
+
 #define APPS_TEMP_PATH "fs:/vol/external01/wiiu/apps/"
-#define RPX_TEMP_FILE       "fs:/vol/external01/wiiu/apps/temp.rpx"
-#define WUHB_TEMP_FILE      "fs:/vol/external01/wiiu/apps/temp.wuhb"
-#define WUHB_TEMP_FILE_2    "fs:/vol/external01/wiiu/apps/temp2.wuhb"
-#define RPX_TEMP_FILE_EX    "wiiu/apps/temp.rpx"
-#define WUHB_TEMP_FILE_EX   "wiiu/apps/temp.wuhb"
-#define WUHB_TEMP_FILE_2_EX "wiiu/apps/temp2.wuhb"
+#define TEMP_FILE_1    "fs:/vol/external01/wiiu/apps/wiiload1.tmp"
+#define TEMP_FILE_2    "fs:/vol/external01/wiiu/apps/wiiload2.tmp"
+#define TEMP_FILE_3    "fs:/vol/external01/wiiu/apps/wiiload3.tmp"
+#define TEMP_FILE_4    "fs:/vol/external01/wiiu/apps/wiiload4.tmp"
+
+static std::string USED_FILE_PATH = "";
+
+namespace {
+    class MemoryStreamFixedSize : public ReadWriteStreamIF {
+    public:
+        explicit MemoryStreamFixedSize(const size_t size)
+            : ReadWriteStreamIF(), mBuffer(make_unique_nothrow<uint8_t[]>(size)), mSize(mBuffer ? size : 0), mPosition(0) {}
+
+
+        MemoryStreamFixedSize(const MemoryStreamFixedSize &) = delete;
+        MemoryStreamFixedSize &operator=(const MemoryStreamFixedSize &) = delete;
+
+        bool isOpen() override {
+            return mBuffer != nullptr;
+        }
+
+        bool isFile() override {
+            return false;
+        }
+
+        bool isDataWrapper() override {
+            return true;
+        }
+
+        std::string getPath() override {
+            return "";
+        }
+        void *getData() override {
+            return mBuffer.get();
+        }
+
+        size_t getSize() override {
+            return mSize;
+        }
+
+        // Write data to the memory stream
+        int64_t write(const void *data, const size_t size) override {
+            size_t copySize = size;
+            if (mPosition + size > mSize) {
+                copySize = mSize - mPosition;
+            }
+            if (copySize == 0) {
+                return 0;
+            }
+            std::memcpy(mBuffer.get() + mPosition, data, size);
+            mPosition += size;
+            return size;
+        }
+
+        // Read data from the memory stream
+        int64_t read(void *output, const size_t size) override {
+            size_t readSize = size;
+            if (mPosition + size > mSize) {
+                readSize = mSize - mPosition;
+            }
+            if (readSize == 0) {
+                return 0;
+            }
+            std::memcpy(output, mBuffer.get() + mPosition, size);
+            mPosition += size;
+            return size;
+        }
+
+        // Seek to a specific position
+        off_t seek(const int64_t offset, const int whence) override {
+            int64_t newPos = mPosition;
+
+            if (whence == SEEK_SET) {
+                newPos = offset;
+            } else if (whence == SEEK_CUR) {
+                newPos += offset;
+            } else if (whence == SEEK_END) {
+                newPos = mSize + offset;
+            }
+            if (newPos < 0) {
+                mPosition = 0;
+            } else {
+                mPosition = newPos;
+            }
+            if (mPosition > mSize) {
+                mPosition = mSize;
+            }
+
+            return mPosition;
+        }
+
+        // Get the current position
+        [[nodiscard]] off_t tell() const override {
+            return mPosition;
+        }
+
+    private:
+        std::unique_ptr<uint8_t[]> mBuffer;
+        size_t mSize;
+        size_t mPosition;
+    };
+
+    class FileStream final : public ReadWriteStreamIF {
+    public:
+        explicit FileStream(const std::string_view path, const std::string_view mode) : mPath(path) {
+            mFile = ::fopen(path.data(), mode.data());
+            if (mFile != nullptr) {
+                setvbuf(mFile, nullptr, _IOFBF, 128 * 1024);
+                ::fseek(mFile, 0, SEEK_END);
+                mSize = ::ftell(mFile);
+                ::fseek(mFile, 0, SEEK_SET);
+                mPosition = 0;
+            }
+        }
+
+        ~FileStream() override {
+            releaseFile();
+        }
+
+        FileStream(const FileStream &) = delete;
+        FileStream &operator=(const FileStream &) = delete;
+
+        FileStream(FileStream &&other) noexcept {
+            releaseFile();
+            mFile           = other.mFile;
+            mPath           = std::move(other.mPath);
+            mSize           = other.mSize;
+            mPosition       = other.mPosition;
+            other.mFile     = nullptr;
+            other.mSize     = 0;
+            other.mPosition = 0;
+        }
+
+        void releaseFile() {
+            if (mFile != nullptr) {
+                ::fclose(mFile);
+                mFile = nullptr;
+            }
+        }
+
+        FileStream &operator=(FileStream &&other) noexcept {
+            if (this != &other) {
+                releaseFile();
+                mFile           = other.mFile;
+                mPath           = std::move(other.mPath);
+                mSize           = other.mSize;
+                mPosition       = other.mPosition;
+                other.mFile     = nullptr;
+                other.mSize     = 0;
+                other.mPosition = 0;
+            }
+            return *this;
+        }
+
+        bool isOpen() override {
+            return mFile != nullptr;
+        }
+
+        bool isFile() override {
+            return true;
+        }
+
+        bool isDataWrapper() override {
+            return false;
+        }
+
+        std::string getPath() override {
+            return mPath;
+        }
+
+        void *getData() override {
+            return nullptr;
+        }
+
+        size_t getSize() override {
+            return mSize;
+        }
+
+        // Write data to the memory stream
+        int64_t write(const void *data, const size_t size) override {
+            const auto res = ::fwrite(data, 1, size, mFile);
+            mPosition += res;
+            if (mPosition > mSize) {
+                mSize = mPosition;
+            }
+            return res;
+        }
+
+        // Read data from the memory stream
+        int64_t read(void *output, const size_t size) override {
+            const auto res = ::fread(output, 1, size, mFile);
+            mPosition += res;
+            return res;
+        }
+
+        // Seek to a specific position
+        off_t seek(const int64_t offset, const int whence) override {
+            const auto res = ::fseek(mFile, offset, whence);
+            mPosition      = res;
+            return res;
+        }
+
+        // Get the current position
+        [[nodiscard]] off_t tell() const override {
+            return mPosition;
+        }
+
+    private:
+        FILE *mFile = nullptr;
+        std::string mPath;
+        size_t mSize     = 0;
+        size_t mPosition = 0;
+    };
+
+    std::unique_ptr<ReadWriteStreamIF> getNewFileOutputStream() {
+        std::unique_ptr<ReadWriteStreamIF> dataStream;
+        if (USED_FILE_PATH != TEMP_FILE_1) {
+            if (dataStream = std::make_unique<FileStream>(TEMP_FILE_1, "w+"); dataStream->isOpen()) {
+                return dataStream;
+            }
+        }
+        if (USED_FILE_PATH != TEMP_FILE_2) {
+            if (dataStream = std::make_unique<FileStream>(TEMP_FILE_2, "w+"); dataStream->isOpen()) {
+                return dataStream;
+            }
+        }
+        if (USED_FILE_PATH != TEMP_FILE_3) {
+            if (dataStream = std::make_unique<FileStream>(TEMP_FILE_3, "w+"); dataStream->isOpen()) {
+                return dataStream;
+            }
+        }
+        if (USED_FILE_PATH != TEMP_FILE_4) {
+            if (dataStream = std::make_unique<FileStream>(TEMP_FILE_4, "w+"); dataStream->isOpen()) {
+                return dataStream;
+            }
+        }
+        return {};
+    }
+
+    bool saveDataToFile(const std::unique_ptr<ReadWriteStreamIF> &inputStream, std::string &loadedPathOut, const NotificationModuleHandle notificationHandle) {
+        if (USED_FILE_PATH != TEMP_FILE_1 && FSUtils::saveBufferToFile(TEMP_FILE_1, inputStream->getData(), inputStream->getSize(), notificationHandle)) {
+            loadedPathOut = TEMP_FILE_1;
+            return true;
+        }
+        if (USED_FILE_PATH != TEMP_FILE_2 && FSUtils::saveBufferToFile(TEMP_FILE_2, inputStream->getData(), inputStream->getSize(), notificationHandle)) {
+            loadedPathOut = TEMP_FILE_2;
+            return true;
+        }
+        if (USED_FILE_PATH != TEMP_FILE_3 && FSUtils::saveBufferToFile(TEMP_FILE_3, inputStream->getData(), inputStream->getSize(), notificationHandle)) {
+            loadedPathOut = TEMP_FILE_3;
+            return true;
+        }
+        if (USED_FILE_PATH != TEMP_FILE_4 && FSUtils::saveBufferToFile(TEMP_FILE_4, inputStream->getData(), inputStream->getSize(), notificationHandle)) {
+            loadedPathOut = TEMP_FILE_4;
+            return true;
+        }
+        return false;
+    }
+
+} // namespace
 
 TcpReceiver::TcpReceiver(int32_t port)
     : CThread(CThread::eAttributeAffCore1, 16, 0x20000, nullptr, nullptr, "Wiiload Thread"), exitRequested(false), serverPort(port), serverSocket(-1) {
@@ -94,6 +357,7 @@ void TcpReceiver::executeThread() {
         if (clientSocket >= 0) {
             uint32_t ipAddress = clientAddr.sin_addr.s_addr;
             DEBUG_FUNCTION_LINE("Waiting for wiiload connection");
+
             auto result = loadToMemory(clientSocket, ipAddress);
             close(clientSocket);
 
@@ -127,6 +391,9 @@ void TcpReceiver::executeThread() {
                 case NO_ACTIVE_ACCOUNT:
                     NotificationModule_AddErrorNotification("Wiiload plugin: Failed to launch homebrew. No active account loaded.");
                     break;
+                case STREAM_ERROR:
+                    NotificationModule_AddErrorNotification("Wiiload plugin: Failed to launch homebrew. Invalid stream type");
+                    break;
             }
 
             if (result == SUCCESS) {
@@ -145,51 +412,100 @@ void TcpReceiver::executeThread() {
     DEBUG_FUNCTION_LINE("Stopping wiiload server.");
 }
 
-TcpReceiver::eLoadResults TcpReceiver::tryLoadWUHB(void *data, uint32_t fileSize, std::string &loadedPathOut) {
-    if (memcmp(data, "WUHB", 4) == 0) {
+TcpReceiver::eLoadResults TcpReceiver::tryLoadWUHB(std::unique_ptr<ReadWriteStreamIF> &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut) {
+    inputStream->seek(0, SEEK_SET);
+    std::vector<uint8_t> header(4);
+    inputStream->read(header.data(), 4);
+    if (memcmp(header.data(), "WUHB", 4) == 0) {
         DEBUG_FUNCTION_LINE("Try to load a .wuhb");
+        if (inputStream->isDataWrapper() && inputStream->getData()) {
             if (!FSUtils::CreateSubfolder(APPS_TEMP_PATH)) {
                 DEBUG_FUNCTION_LINE_WARN("Failed to create directory: %s", APPS_TEMP_PATH);
                 return FILE_SAVE_BUFFER_ERROR;
             }
-        if (FSUtils::saveBufferToFile(WUHB_TEMP_FILE, data, fileSize)) {
-            loadedPathOut = WUHB_TEMP_FILE_EX;
-        } else if (FSUtils::saveBufferToFile(WUHB_TEMP_FILE_2, data, fileSize)) {
-            loadedPathOut = WUHB_TEMP_FILE_2_EX;
-        } else {
+            if (!saveDataToFile(inputStream, loadedPathOut, notificationHandle)) {
                 DEBUG_FUNCTION_LINE_WARN("Failed to save .wuhb file to the sd card. Launching will be aborted.");
                 return FILE_SAVE_BUFFER_ERROR;
             }
+        } else if (inputStream->isFile()) {
+            loadedPathOut = inputStream->getPath();
+
+            if (loadedPathOut.empty()) {
+                DEBUG_FUNCTION_LINE_WARN("Path was empty. Launching will be aborted.");
+                return FILE_SAVE_BUFFER_ERROR;
+            }
+            // Release stream so we can access it!
+            inputStream.reset();
+        } else {
+            DEBUG_FUNCTION_LINE_WARN("Invalid stream type detected. Launching will be aborted.");
+            return FILE_SAVE_BUFFER_ERROR;
+        }
         return SUCCESS;
     }
     DEBUG_FUNCTION_LINE_VERBOSE("Loaded data is not a WUHB");
     return UNSUPPORTED_FORMAT;
 }
 
-TcpReceiver::eLoadResults TcpReceiver::tryLoadRPX(uint8_t *data, uint32_t fileSize, std::string &loadedPathOut) {
-    if (data[0x7] == 0xCA && data[0x8] == 0xFE && data[0x9] != 0x50 && data[0xA] != 0x4C) {
+TcpReceiver::eLoadResults TcpReceiver::tryLoadRPX(std::unique_ptr<ReadWriteStreamIF> &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut) {
+    inputStream->seek(0, SEEK_SET);
+    std::vector<uint8_t> header(16);
+    inputStream->read(header.data(), 16);
+    if (header[0x7] == 0xCA && header[0x8] == 0xFE && header[0x9] != 0x50 && header[0xA] != 0x4C) {
         DEBUG_FUNCTION_LINE("Try to load a .rpx");
+        if (inputStream->isDataWrapper() && inputStream->getData()) {
             if (!FSUtils::CreateSubfolder(APPS_TEMP_PATH)) {
                 DEBUG_FUNCTION_LINE_WARN("Failed to create directory: %s", APPS_TEMP_PATH);
                 return FILE_SAVE_BUFFER_ERROR;
             }
-        if (FSUtils::saveBufferToFile(RPX_TEMP_FILE, data, fileSize)) {
-            loadedPathOut = RPX_TEMP_FILE_EX;
-        } else {
+            if (!saveDataToFile(inputStream, loadedPathOut, notificationHandle)) {
                 DEBUG_FUNCTION_LINE_WARN("Failed to save .rpx file to the sd card. Launching will be aborted.");
                 return FILE_SAVE_BUFFER_ERROR;
             }
+        } else if (inputStream->isFile()) {
+            loadedPathOut = inputStream->getPath();
+
+            if (loadedPathOut.empty()) {
+                DEBUG_FUNCTION_LINE_WARN("Path was empty. Launching will be aborted.");
+                return FILE_SAVE_BUFFER_ERROR;
+            }
+            // Release stream so we can access it!
+            inputStream.reset();
+        } else {
+            DEBUG_FUNCTION_LINE_WARN("Invalid stream type detected. Launching will be aborted.");
+            return FILE_SAVE_BUFFER_ERROR;
+        }
         return SUCCESS;
     }
     DEBUG_FUNCTION_LINE_VERBOSE("Loaded data is not a RPX");
     return UNSUPPORTED_FORMAT;
 }
 
-TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSize) {
-    if (data[0x7] == 0xCA && data[0x8] == 0xFE && data[0x9] == 0x50 && data[0xA] == 0x4C) {
+TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(std::unique_ptr<ReadWriteStreamIF> &inputStream, NotificationModuleHandle notificationHandle) {
+    inputStream->seek(0, SEEK_SET);
+    std::vector<uint8_t> header(16);
+    inputStream->read(header.data(), 16);
+    if (header[0x7] == 0xCA && header[0x8] == 0xFE && header[0x9] == 0x50 && header[0xA] == 0x4C) {
         PluginBackendApiErrorType err;
         PluginBackendPluginParseError parseErr;
-        auto newContainerOpt = WUPSBackend::PluginUtils::getPluginForBuffer((char *) data, fileSize, err, parseErr);
+        std::optional<WUPSBackend::PluginContainer> newContainerOpt;
+        if (inputStream->isFile()) {
+            const auto path = inputStream->getPath();
+            // Reset stream ptr so we can access the file!
+            inputStream.reset();
+
+            newContainerOpt = WUPSBackend::PluginUtils::getPluginForPath(path, err, parseErr);
+            // We can delete file no from the sd card
+            remove(path.c_str());
+        } else if (inputStream->isDataWrapper()) {
+            DEBUG_FUNCTION_LINE_ERR("Loading from buffer");
+            newContainerOpt = WUPSBackend::PluginUtils::getPluginForBuffer(static_cast<char *>(inputStream->getData()), inputStream->getSize(), err, parseErr);
+            // release the memory as early as possible.
+            inputStream.reset();
+        } else {
+            DEBUG_FUNCTION_LINE_ERR("Invalid stream type. This should never happen!");
+            return STREAM_ERROR;
+        }
+
         if (newContainerOpt) {
             auto pluginList = WUPSBackend::PluginUtils::getLoadedPlugins(err);
             if (err != PLUGIN_BACKEND_API_ERROR_NONE) {
@@ -199,13 +515,12 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSi
 
             const auto &metaInformation = newContainerOpt->getMetaInformation();
             // remove plugins with the same name and author as our new plugin
-            pluginList.erase(std::remove_if(pluginList.begin(), pluginList.end(),
+            std::erase_if(pluginList,
                           [&metaInformation](auto &plugin) {
                               auto res = plugin.getMetaInformation().getName() == metaInformation.getName() &&
                                          plugin.getMetaInformation().getAuthor() == metaInformation.getAuthor();
                               return res;
-                                            }),
-                             pluginList.end());
+                          });
             // add the new plugin
             pluginList.push_back(std::move(newContainerOpt.value()));
 #ifdef VERBOSE_DEBUG
@@ -222,7 +537,7 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSi
             }
             return SUCCESS;
         } else {
-            DEBUG_FUNCTION_LINE_ERR("Failed to parse plugin for buffer: %08X size %d. Error: %s", data, fileSize, WUPSBackend::GetStatusStr(err));
+            DEBUG_FUNCTION_LINE_ERR("Failed to parse plugin. Error: %s", WUPSBackend::GetStatusStr(err));
             return PLUGIN_PARSE_FAILED;
         }
     }
@@ -230,14 +545,23 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSi
     return UNSUPPORTED_FORMAT;
 }
 
-TcpReceiver::eLoadResults TcpReceiver::loadBinary(void *data, uint32_t fileSize) {
+TcpReceiver::eLoadResults TcpReceiver::loadBinary(std::unique_ptr<ReadWriteStreamIF> &&inputStream, const NotificationModuleHandle notificationHandle) {
     std::string loadedPath;
-    eLoadResults error;
-    if ((error = tryLoadWUHB(data, fileSize, loadedPath)) != UNSUPPORTED_FORMAT || (error = tryLoadRPX((uint8_t *) data, fileSize, loadedPath)) != UNSUPPORTED_FORMAT) {
+    if (eLoadResults error; (error = tryLoadWUHB(inputStream, notificationHandle, loadedPath)) != UNSUPPORTED_FORMAT || (error = tryLoadRPX(inputStream, notificationHandle, loadedPath)) != UNSUPPORTED_FORMAT) {
         if (error != SUCCESS) {
             return error;
         }
 
+        // lets save which file we don't want to override
+        USED_FILE_PATH = loadedPath;
+
+        const auto index = loadedPath.find("fs:/vol/external01/");
+        if (index != std::string::npos) {
+            loadedPath.erase(index, strlen("fs:/vol/external01/"));
+        }
+
+        DEBUG_FUNCTION_LINE_VERBOSE("Loaded file: other %s", loadedPath.c_str());
+
         nn::act::Initialize();
         bool accountLoaded = nn::act::IsSlotOccupied(nn::act::GetSlotNo());
         nn::act::Finalize();
@@ -246,123 +570,173 @@ TcpReceiver::eLoadResults TcpReceiver::loadBinary(void *data, uint32_t fileSize)
             return NO_ACTIVE_ACCOUNT;
         }
 
-        RPXLoaderStatus launchRes;
-        if ((launchRes = RPXLoader_LaunchHomebrew(loadedPath.c_str())) != RPX_LOADER_RESULT_SUCCESS) {
-            DEBUG_FUNCTION_LINE_ERR("Failed to start %s %s", loadedPath.c_str(), RPXLoader_GetStatusStr(launchRes));
+        if (const RPXLoaderStatus launchRes = RPXLoader_LaunchHomebrew(loadedPath.c_str()); launchRes != RPX_LOADER_RESULT_SUCCESS) {
+            DEBUG_FUNCTION_LINE_ERR("Failed to start \"%s\": %s", loadedPath.c_str(), RPXLoader_GetStatusStr(launchRes));
             return LAUNCH_FAILED;
         }
-    } else if ((error = tryLoadWPS((uint8_t *) data, fileSize)) != UNSUPPORTED_FORMAT) {
+
+        NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Launching homebrew...");
+        return SUCCESS;
+    } else if ((error = tryLoadWPS(inputStream, notificationHandle)) != UNSUPPORTED_FORMAT) {
         if (error != SUCCESS) {
             return error;
         }
         _SYSLaunchTitleWithStdArgsInNoSplash(OSGetTitleID(), nullptr);
-    } else {
-        return UNSUPPORTED_FORMAT;
-    }
+        NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Reloading with new plugin...");
         return SUCCESS;
     }
-
-std::unique_ptr<uint8_t[]> TcpReceiver::uncompressData(uint32_t fileSize, uint32_t fileSizeUnc, std::unique_ptr<uint8_t[]> &&inData, uint32_t &fileSizeOut, eLoadResults &err) {
-    std::unique_ptr<uint8_t[]> inflatedData;
-    uint8_t *in_data_raw = inData.get();
-    // We need to unzip...
-    if (in_data_raw[0] == 'P' && in_data_raw[1] == 'K' && in_data_raw[2] == 0x03 && in_data_raw[3] == 0x04) {
-        // Section is compressed, inflate
-        inflatedData = make_unique_nothrow<uint8_t[]>(fileSizeUnc);
-        if (!inflatedData) {
-            DEBUG_FUNCTION_LINE_ERR("malloc failed");
-            err = NOT_ENOUGH_MEMORY;
-            return {};
+    return UNSUPPORTED_FORMAT;
 }
 
-        int32_t ret;
-        z_stream s = {};
+constexpr size_t CHUNK_SIZE = 128 * 1024; // Size of chunks to process
 
-        s.zalloc = Z_NULL;
-        s.zfree  = Z_NULL;
-        s.opaque = Z_NULL;
-
-        ret = inflateInit(&s);
-        if (ret != Z_OK) {
-            DEBUG_FUNCTION_LINE_ERR("inflateInit failed %i", ret);
-            err = FILE_UNCOMPRESS_ERROR;
+std::unique_ptr<ReadWriteStreamIF> TcpReceiver::uncompressData(uint32_t fileSizeUnc, std::unique_ptr<ReadWriteStreamIF> &&inputStream, const NotificationModuleHandle notificationHandle, const bool toFile, eLoadResults &err) {
+    if (!inputStream->isOpen()) {
+        DEBUG_FUNCTION_LINE_ERR("inputStream is not open");
         return {};
     }
-
-        s.avail_in = fileSize;
-        s.next_in  = (Bytef *) inflatedData.get();
-
-        s.avail_out = fileSizeUnc;
-        s.next_out  = (Bytef *) inflatedData.get();
-
-        ret = inflate(&s, Z_FINISH);
-        if (ret != Z_OK && ret != Z_STREAM_END) {
-            DEBUG_FUNCTION_LINE_ERR("inflate failed %i", ret);
-            err = FILE_UNCOMPRESS_ERROR;
-            return {};
-        }
-
-        inflateEnd(&s);
+    std::unique_ptr<ReadWriteStreamIF> outputStream = {};
+    if (toFile) {
+        outputStream = getNewFileOutputStream();
     } else {
-        // Section is compressed, inflate
-        inflatedData = make_unique_nothrow<uint8_t[]>(fileSizeUnc);
-        if (!inflatedData) {
-            DEBUG_FUNCTION_LINE_ERR("malloc failed");
-            err = NOT_ENOUGH_MEMORY;
+        outputStream = std::make_unique<MemoryStreamFixedSize>(fileSizeUnc);
+    }
+    if (!outputStream || !outputStream->isOpen()) {
+        DEBUG_FUNCTION_LINE_ERR("Failed to open or create outputStream");
         return {};
     }
 
-        uLongf f       = fileSizeUnc;
-        int32_t result = uncompress((Bytef *) inflatedData.get(), &f, (Bytef *) in_data_raw, fileSize);
-        if (result != Z_OK) {
-            DEBUG_FUNCTION_LINE_ERR("uncompress failed %i", result);
-            err = FILE_UNCOMPRESS_ERROR;
+    inputStream->seek(0, SEEK_SET);
+    outputStream->seek(0, SEEK_SET);
+
+    std::vector<unsigned char> inBuffer(CHUNK_SIZE);
+    std::vector<unsigned char> outBuffer(CHUNK_SIZE);
+
+    z_stream strm = {};
+    strm.zalloc   = Z_NULL;
+    strm.zfree    = Z_NULL;
+    strm.opaque   = Z_NULL;
+
+    // Initialize the inflate state
+    if (inflateInit(&strm) != Z_OK) {
+        DEBUG_FUNCTION_LINE_ERR("Failed to call inflateInit");
         return {};
     }
 
-        fileSizeUnc = f;
+    int ret = Z_STREAM_ERROR;
+    do {
+        // Read a chunk of compressed data
+        auto readRes = inputStream->read(inBuffer.data(), CHUNK_SIZE);
+
+        std::string progressStr = string_format("[Wiiload] Decompressing data... %.2f%%", static_cast<float>(inputStream->tell()) / static_cast<float>(inputStream->getSize()) * 100.0f);
+        NotificationModule_UpdateDynamicNotificationText(notificationHandle, progressStr.c_str());
+
+        if (readRes <= 0) {
+            break;
         }
 
-    fileSizeOut = fileSizeUnc;
-    err         = SUCCESS;
-    return inflatedData;
+        strm.avail_in = readRes;
+        strm.next_in  = inBuffer.data();
+
+        // Decompress in a loop
+        do {
+            strm.avail_out = CHUNK_SIZE;
+            strm.next_out  = outBuffer.data();
+
+            ret = inflate(&strm, Z_NO_FLUSH);
+            assert(ret != Z_STREAM_ERROR); /* state not clobbered */
+            switch (ret) {
+                case Z_NEED_DICT:
+                    ret = Z_DATA_ERROR; /* and fall through */
+                case Z_DATA_ERROR:
+                case Z_MEM_ERROR: {
+                    DEBUG_FUNCTION_LINE_ERR("inflate() failed");
+                    (void) inflateEnd(&strm);
+                    return {};
+                }
             }
 
-std::unique_ptr<uint8_t[]> TcpReceiver::receiveData(int32_t clientSocket, uint32_t fileSize, eLoadResults &err) {
+            auto have = CHUNK_SIZE - strm.avail_out;
+            if (outputStream->write(outBuffer.data(), have) != have) {
+                DEBUG_FUNCTION_LINE_ERR("write() failed");
+                (void) inflateEnd(&strm);
+                return {};
+            }
+        } while (strm.avail_out == 0);
+
+    } while (ret != Z_STREAM_END);
+
+    // clean up uncompressed file
+    if (inputStream->isFile()) {
+        const auto path = inputStream->getPath();
+        inputStream.reset();
+        remove(path.c_str());
+    }
+
+    // Clean up
+    inflateEnd(&strm);
+    if (ret != Z_STREAM_END) {
+        DEBUG_FUNCTION_LINE_ERR("Failed to uncompress data");
+        NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Decompressing data... failed");
+        return {};
+    }
+
+    NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Decompressing data... 100.00%");
+    return outputStream;
+}
+
+bool TcpReceiver::receiveData(const int32_t clientSocket, const std::unique_ptr<ReadWriteStreamIF> &output, const uint32_t fileSize, const NotificationModuleHandle notificationHandle, eLoadResults &err) {
     uint32_t bytesRead = 0;
-    auto dataOut       = make_unique_nothrow<uint8_t[]>(fileSize);
-    if (!dataOut) {
-        err = NOT_ENOUGH_MEMORY;
-        return {};
-    }
-
+    std::vector<unsigned char> inBuffer(CHUNK_SIZE);
     // Copy binary in memory
     while (bytesRead < fileSize) {
-        uint32_t blockSize = 0x1000;
+        std::string progressStr = string_format("[Wiiload] Receiving data... %.2f%%", static_cast<float>(bytesRead) / static_cast<float>(fileSize) * 100.0f);
+        NotificationModule_UpdateDynamicNotificationText(notificationHandle, progressStr.c_str());
+
+        uint32_t blockSize = CHUNK_SIZE;
         if (blockSize > (fileSize - bytesRead))
             blockSize = fileSize - bytesRead;
 
-        int32_t ret = recv(clientSocket, dataOut.get() + bytesRead, blockSize, 0);
+        const int32_t ret = recv(clientSocket, inBuffer.data(), blockSize, 0);
         if (ret <= 0) {
             DEBUG_FUNCTION_LINE_ERR("Failed to receive file");
             break;
         }
+        int64_t writtenData = 0;
+        while (writtenData < ret) {
+            const int64_t toWrite = ret - writtenData;
+            const auto res        = output->write(inBuffer.data() + writtenData, toWrite);
+            if (res <= 0) {
+                if (res < 0) {
+                    DEBUG_FUNCTION_LINE_ERR("Failed to receive file");
+                }
+                break;
+            }
+            writtenData += res;
+        }
+        if (writtenData != ret) {
+            DEBUG_FUNCTION_LINE_ERR("Failed to write to file");
+            break;
+        }
 
         bytesRead += ret;
     }
 
     if (bytesRead != fileSize) {
+        NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Receiving data... failed");
         DEBUG_FUNCTION_LINE_ERR("File loading not finished, %i of %i bytes received", bytesRead, fileSize);
         err = RECV_ERROR;
-        return {};
+        return false;
     }
+
+    NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Receiving data... 100.00%");
+
     err = SUCCESS;
-    return dataOut;
+    return true;
 }
 
 TcpReceiver::eLoadResults TcpReceiver::loadToMemory(int32_t clientSocket, uint32_t ipAddress) {
     DEBUG_FUNCTION_LINE("Loading file from ip %08X", ipAddress);
-
     uint32_t fileSize    = 0;
     uint32_t fileSizeUnc = 0;
     uint8_t haxx[8]      = {};
@@ -379,16 +753,38 @@ TcpReceiver::eLoadResults TcpReceiver::loadToMemory(int32_t clientSocket, uint32
             return RECV_ERROR;
         }
     }
-    TcpReceiver::eLoadResults err = UNSUPPORTED_FORMAT;
-    auto receivedData             = receiveData(clientSocket, fileSize, err);
-    if (err != SUCCESS) {
-        return err;
-    } else if (compressedData) {
-        receivedData = uncompressData(fileSize, fileSizeUnc, std::move(receivedData), fileSize, err);
-        if (!receivedData || err != SUCCESS) {
-            return err;
-        }
+    eLoadResults err = UNSUPPORTED_FORMAT;
+
+    NotificationModuleHandle notificationHandle;
+    if (NotificationModule_AddDynamicNotification("[Wiiload] Receiving binary via network", &notificationHandle) != NOTIFICATION_MODULE_RESULT_SUCCESS) {
+        notificationHandle = 0;
     }
 
-    return loadBinary(receivedData.get(), fileSize);
+    const bool asFile = fileSize > 1;
+    std::unique_ptr<ReadWriteStreamIF> dataStream;
+    if (asFile) {
+        dataStream = getNewFileOutputStream();
+    } else {
+        dataStream = std::make_unique<MemoryStreamFixedSize>(fileSize);
+    }
+
+    if (!dataStream || !receiveData(clientSocket, dataStream, fileSize, notificationHandle, err) || err != SUCCESS) {
+        DEBUG_FUNCTION_LINE_ERR("ERROR: Failed to receive file");
+        NotificationModule_FinishDynamicNotification(notificationHandle, 0.5f);
+        return err;
+    }
+
+    if (compressedData) {
+        dataStream = uncompressData(fileSizeUnc, std::move(dataStream), notificationHandle, asFile, err);
+        if (!dataStream || err != SUCCESS) {
+            DEBUG_FUNCTION_LINE_ERR("ERROR: Failed to uncompress file");
+            NotificationModule_FinishDynamicNotification(notificationHandle, 0.5f);
+            return err;
+        }
+    }
+    dataStream->seek(0, SEEK_SET);
+
+    const auto res = loadBinary(std::move(dataStream), notificationHandle);
+    NotificationModule_FinishDynamicNotification(notificationHandle, 0.5f);
+    return res;
 }

src/utils/TcpReceiver.h

@@ -1,10 +1,13 @@
 #pragma once
 
+#include "CThread.h"
+
+#include <notifications/notification_defines.h>
+
 #include <memory>
 #include <string>
-#include <vector>
 
-#include "CThread.h"
+class ReadWriteStreamIF;
 
 class TcpReceiver : public CThread {
 public:
@@ -19,6 +22,7 @@ public:
         RECV_ERROR              = -7,
         LAUNCH_FAILED           = -8,
         NO_ACTIVE_ACCOUNT       = -9,
+        STREAM_ERROR            = -10,
     };
 
     explicit TcpReceiver(int32_t port);
@@ -33,12 +37,12 @@ private:
 
     static eLoadResults loadToMemory(int32_t clientSocket, uint32_t ipAddress);
 
-    static TcpReceiver::eLoadResults tryLoadWUHB(void *data, uint32_t fileSize, std::string &loadedPathOut);
-    static TcpReceiver::eLoadResults tryLoadRPX(uint8_t *data, uint32_t fileSize, std::string &loadedPathOut);
-    static TcpReceiver::eLoadResults tryLoadWPS(uint8_t *data, uint32_t fileSize);
-    static TcpReceiver::eLoadResults loadBinary(void *data, uint32_t fileSize);
-    static std::unique_ptr<uint8_t[]> receiveData(int32_t clientSocket, uint32_t fileSize, eLoadResults &err);
-    static std::unique_ptr<uint8_t[]> uncompressData(uint32_t fileSize, uint32_t fileSizeUnc, std::unique_ptr<uint8_t[]> &&in_out_data, uint32_t &fileSizeOut, eLoadResults &err);
+    static TcpReceiver::eLoadResults tryLoadWUHB(std::unique_ptr<ReadWriteStreamIF> &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut);
+    static TcpReceiver::eLoadResults tryLoadRPX(std::unique_ptr<ReadWriteStreamIF> &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut);
+    static TcpReceiver::eLoadResults tryLoadWPS(std::unique_ptr<ReadWriteStreamIF> &inputStream, NotificationModuleHandle notificationHandle);
+    static TcpReceiver::eLoadResults loadBinary(std::unique_ptr<ReadWriteStreamIF> &&inputStream, NotificationModuleHandle notificationHandle);
+    static bool receiveData(int32_t clientSocket, const std::unique_ptr<ReadWriteStreamIF> &output, uint32_t fileSize, NotificationModuleHandle notificationHandle, eLoadResults &err);
+    static std::unique_ptr<ReadWriteStreamIF> uncompressData(uint32_t fileSizeUnc, std::unique_ptr<ReadWriteStreamIF> &&inputStream, NotificationModuleHandle notificationHandle, bool toFile, eLoadResults &err);
 
     bool exitRequested;
     int32_t serverPort;

src/utils/net.cpp

@@ -1,28 +1,25 @@
 #include "net.h"
+
+#include <mutex>
+
 #include <unistd.h>
 
-static volatile int socket_lock __attribute__((section(".data"))) = 0;
+static std::mutex sSocketMutex;
 
-int32_t recvwait(int32_t sock, void *buffer, int32_t len) {
-    while (socket_lock) {
-        usleep(1000);
-    }
-    socket_lock = 1;
-    int32_t ret;
+int32_t recvwait(const int32_t sock, void *buffer, int32_t len) {
+    std::lock_guard lock(sSocketMutex);
     while (len > 0) {
-        ret = recv(sock, buffer, len, 0);
+        const int32_t ret = recv(sock, buffer, len, 0);
         if (ret < 0) {
-            socket_lock = 0;
             return ret;
         }
         len -= ret;
         buffer = (void *) (((char *) buffer) + ret);
     }
-    socket_lock = 0;
     return 0;
 }
 
-uint8_t recvbyte(int32_t sock) {
+uint8_t recvbyte(const int32_t sock) {
     unsigned char buffer[1];
     int32_t ret;
 
@@ -32,7 +29,7 @@ uint8_t recvbyte(int32_t sock) {
     return buffer[0];
 }
 
-uint32_t recvword(int32_t sock) {
+uint32_t recvword(const int32_t sock) {
     uint32_t result;
     int32_t ret;
 
@@ -42,16 +39,11 @@ uint32_t recvword(int32_t sock) {
     return result;
 }
 
-int32_t checkbyte(int32_t sock) {
-    while (socket_lock) {
-        usleep(1000);
-    }
-    socket_lock = 1;
+int32_t checkbyte(const int32_t sock) {
+    std::lock_guard lock(sSocketMutex);
     unsigned char buffer[1];
-    int32_t ret;
 
-    ret         = recv(sock, buffer, 1, MSG_DONTWAIT);
-    socket_lock = 0;
+    const int32_t ret = recv(sock, buffer, 1, MSG_DONTWAIT);
     if (ret < 0)
         return ret;
     if (ret == 0)
@@ -60,23 +52,17 @@ int32_t checkbyte(int32_t sock) {
 }
 
 int32_t sendwait(int32_t sock, const void *buffer, int32_t len) {
-    while (socket_lock) {
-        usleep(1000);
-    }
-    socket_lock = 1;
-    int32_t ret;
+    std::lock_guard lock(sSocketMutex);
     while (len > 0) {
-        // For some reason the send blocks/crashes if the buffer is too big..
-        int cur_length = len <= 0x30 ? len : 0x30;
-        ret            = send(sock, buffer, cur_length, 0);
+        // For some reason the send blocks/crashes if the buffer is too big...
+        const int cur_length = len <= 0x30 ? len : 0x30;
+        const int32_t ret    = send(sock, buffer, cur_length, 0);
         if (ret < 0) {
-            socket_lock = 0;
             return ret;
         }
         len -= ret;
         buffer = (void *) (((char *) buffer) + ret);
     }
-    socket_lock = 0;
     return 0;
 }
 

src/utils/utils.h

@@ -3,6 +3,7 @@
 #include <malloc.h>
 #include <memory>
 
+
 template<class T, class... Args>
 std::unique_ptr<T> make_unique_nothrow(Args &&...args) noexcept(noexcept(T(std::forward<Args>(args)...))) {
     return std::unique_ptr<T>(new (std::nothrow) T(std::forward<Args>(args)...));
@@ -12,3 +13,17 @@ template<typename T>
 inline typename std::unique_ptr<T> make_unique_nothrow(size_t num) noexcept {
     return std::unique_ptr<T>(new (std::nothrow) std::remove_extent_t<T>[num]());
 }
+
+template<typename... Args>
+std::string string_format(const std::string_view format, Args... args) {
+    const int size_s = std::snprintf(nullptr, 0, format.data(), args...) + 1; // Extra space for '\0'
+    const auto size  = static_cast<size_t>(size_s);
+    const auto buf   = make_unique_nothrow<char[]>(size);
+    if (!buf) {
+        DEBUG_FUNCTION_LINE_ERR("string_format failed, not enough memory");
+        OSFatal("string_format failed, not enough memory");
+        return std::string("");
+    }
+    std::snprintf(buf.get(), size, format.data(), args...);
+    return std::string(buf.get(), buf.get() + size - 1); // We don't want the '\0' inside
+}