From 88ddd102f38b5afb0f183e9e32292b64b8eb9bd8 Mon Sep 17 00:00:00 2001 From: Maschell Date: Mon, 20 Jan 2025 18:00:56 +0100 Subject: [PATCH] Support loading files which don't fit into memory by saving them to the sd card --- Dockerfile | 4 +- src/config.cpp | 4 +- src/globals.cpp | 1 + src/globals.h | 4 +- src/main.cpp | 5 +- src/utils/CThread.h | 1 + src/utils/FSUtils.cpp | 17 +- src/utils/FSUtils.h | 3 +- src/utils/ReadWriteStreamIF.h | 37 ++ src/utils/TcpReceiver.cpp | 678 +++++++++++++++++++++++++++------- src/utils/TcpReceiver.h | 20 +- src/utils/{net.c => net.cpp} | 46 +-- src/utils/utils.h | 17 +- 13 files changed, 649 insertions(+), 188 deletions(-) create mode 100644 src/utils/ReadWriteStreamIF.h rename src/utils/{net.c => net.cpp} (55%) diff --git a/Dockerfile b/Dockerfile index 9e9e8ad..644f1cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM ghcr.io/wiiu-env/devkitppc:20240505 +FROM ghcr.io/wiiu-env/devkitppc:20241128 COPY --from=ghcr.io/wiiu-env/wiiupluginsystem:20240505 /artifacts $DEVKITPRO COPY --from=ghcr.io/wiiu-env/libwupsbackend:20240425 /artifacts $DEVKITPRO COPY --from=ghcr.io/wiiu-env/librpxloader:20240425 /artifacts $DEVKITPRO -COPY --from=ghcr.io/wiiu-env/libnotifications:20240426 /artifacts $DEVKITPRO +COPY --from=ghcr.io/wiiu-env/libnotifications:20241221 /artifacts $DEVKITPRO WORKDIR project diff --git a/src/config.cpp b/src/config.cpp index 79b40b9..e82daa2 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -3,10 +3,12 @@ #include "utils/TcpReceiver.h" #include "utils/logger.h" #include "utils/utils.h" -#include + #include #include +#include + static void gServerEnabledChanged(ConfigItemBoolean *item, bool newValue) { if (std::string_view(WIILOAD_ENABLED_STRING) != item->identifier) { DEBUG_FUNCTION_LINE_WARN("Unexpected identifier in bool callback: %s", item->identifier); diff --git a/src/globals.cpp b/src/globals.cpp index e972937..0ce838d 100644 --- a/src/globals.cpp +++ b/src/globals.cpp @@ -1,4 +1,5 @@ #include "globals.h" +#include "utils/TcpReceiver.h" bool gLibRPXLoaderInitDone = false; std::unique_ptr gTcpReceiverThread = nullptr; diff --git a/src/globals.h b/src/globals.h index db316e7..ee0eb7d 100644 --- a/src/globals.h +++ b/src/globals.h @@ -1,7 +1,7 @@ -#include "utils/TcpReceiver.h" -#include #include +class TcpReceiver; + extern bool gLibRPXLoaderInitDone; extern std::unique_ptr gTcpReceiverThread; extern bool gWiiloadServerEnabled; diff --git a/src/main.cpp b/src/main.cpp index f7dd395..3cb3b50 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4,9 +4,12 @@ #include "utils/TcpReceiver.h" #include "utils/logger.h" #include "utils/utils.h" -#include + #include #include + +#include + #include #include #include diff --git a/src/utils/CThread.h b/src/utils/CThread.h index 3bce2ed..04ae4bc 100644 --- a/src/utils/CThread.h +++ b/src/utils/CThread.h @@ -20,6 +20,7 @@ #include #include #include +#include #include class CThread { diff --git a/src/utils/FSUtils.cpp b/src/utils/FSUtils.cpp index 79d779a..50e6ed6 100644 --- a/src/utils/FSUtils.cpp +++ b/src/utils/FSUtils.cpp @@ -1,8 +1,13 @@ #include "FSUtils.h" #include "logger.h" +#include "utils.h" + #include #include #include +#include +#include +#include #include bool FSUtils::CheckFile(const char *filepath) { @@ -82,7 +87,7 @@ bool FSUtils::CreateSubfolder(const char *fullpath) { return true; } -bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size) { +bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size, NotificationModuleHandle notificationHandle) { int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY); if (fd < 0) { DEBUG_FUNCTION_LINE_ERR("Failed to open %s. %d", path, fd); @@ -93,6 +98,10 @@ bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size) { int32_t curResult; int64_t totalSizeWritten = 0; while (sizeToWrite > 0) { + if (notificationHandle != 0) { + std::string progressStr = string_format("[Wiiload] Write data to file %.2f%%", static_cast(totalSizeWritten) / static_cast(size) * 100.0); + NotificationModule_UpdateDynamicNotificationText(notificationHandle, progressStr.c_str()); + } curResult = write(fd, ptr, sizeToWrite); if (curResult < 0) { close(fd); @@ -106,5 +115,11 @@ bool FSUtils::saveBufferToFile(const char *path, void *buffer, uint32_t size) { sizeToWrite -= curResult; } close(fd); + + if (notificationHandle != 0) { + std::string progressStr = string_format("[Wiiload] Save data to file %.2f%%", static_cast(totalSizeWritten) / static_cast(size) * 100.0); + NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Write data to file 100%"); + } + return totalSizeWritten == size; } diff --git a/src/utils/FSUtils.h b/src/utils/FSUtils.h index eb796c1..a5169a0 100644 --- a/src/utils/FSUtils.h +++ b/src/utils/FSUtils.h @@ -1,5 +1,6 @@ #pragma once +#include #include class FSUtils { @@ -8,5 +9,5 @@ public: static bool CheckFile(const char *filepath); - static bool saveBufferToFile(const char *path, void *buffer, uint32_t size); + static bool saveBufferToFile(const char *path, void *buffer, uint32_t size, NotificationModuleHandle notificationHandle); }; \ No newline at end of file diff --git a/src/utils/ReadWriteStreamIF.h b/src/utils/ReadWriteStreamIF.h new file mode 100644 index 0000000..93a8b27 --- /dev/null +++ b/src/utils/ReadWriteStreamIF.h @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include + +class ReadWriteStreamIF { +public: + ReadWriteStreamIF() = default; + + virtual ~ReadWriteStreamIF() = default; + // Delete the copy constructor and copy assignment operator + ReadWriteStreamIF(const ReadWriteStreamIF &) = delete; + ReadWriteStreamIF &operator=(const ReadWriteStreamIF &) = delete; + + virtual bool isOpen() = 0; + + // must not return false if a path can be returned + virtual bool isFile() = 0; + virtual std::string getPath() = 0; + + // must not return false if a data/size can be returned + virtual bool isDataWrapper() = 0; + virtual void *getData() = 0; + virtual size_t getSize() = 0; + + virtual int64_t write(const void *data, size_t size) = 0; + + // Read data from the memory stream + virtual int64_t read(void *output, size_t size) = 0; + + // Seek to a specific position + virtual off_t seek(int64_t position, int whence) = 0; + + // Get the current position + [[nodiscard]] virtual off_t tell() const = 0; +}; \ No newline at end of file diff --git a/src/utils/TcpReceiver.cpp b/src/utils/TcpReceiver.cpp index 918d6a5..38cce17 100644 --- a/src/utils/TcpReceiver.cpp +++ b/src/utils/TcpReceiver.cpp @@ -1,29 +1,292 @@ #include "TcpReceiver.h" + #include "FSUtils.h" -#include "globals.h" -#include "utils/net.h" -#include "utils/utils.h" -#include -#include -#include -#include -#include -#include -#include +#include "ReadWriteStreamIF.h" +#include "net.h" +#include "utils.h" + #include #include -#include -#include #include +#include + +#include +#include +#include +#include + #include -#define APPS_TEMP_PATH "fs:/vol/external01/wiiu/apps/" -#define RPX_TEMP_FILE "fs:/vol/external01/wiiu/apps/temp.rpx" -#define WUHB_TEMP_FILE "fs:/vol/external01/wiiu/apps/temp.wuhb" -#define WUHB_TEMP_FILE_2 "fs:/vol/external01/wiiu/apps/temp2.wuhb" -#define RPX_TEMP_FILE_EX "wiiu/apps/temp.rpx" -#define WUHB_TEMP_FILE_EX "wiiu/apps/temp.wuhb" -#define WUHB_TEMP_FILE_2_EX "wiiu/apps/temp2.wuhb" +#include +#include +#include + +#include +#include +#include +#include + +#define APPS_TEMP_PATH "fs:/vol/external01/wiiu/apps/" +#define TEMP_FILE_1 "fs:/vol/external01/wiiu/apps/wiiload1.tmp" +#define TEMP_FILE_2 "fs:/vol/external01/wiiu/apps/wiiload2.tmp" +#define TEMP_FILE_3 "fs:/vol/external01/wiiu/apps/wiiload3.tmp" +#define TEMP_FILE_4 "fs:/vol/external01/wiiu/apps/wiiload4.tmp" + +static std::string USED_FILE_PATH = ""; + +namespace { + class MemoryStreamFixedSize : public ReadWriteStreamIF { + public: + explicit MemoryStreamFixedSize(const size_t size) + : ReadWriteStreamIF(), mBuffer(make_unique_nothrow(size)), mSize(mBuffer ? size : 0), mPosition(0) {} + + + MemoryStreamFixedSize(const MemoryStreamFixedSize &) = delete; + MemoryStreamFixedSize &operator=(const MemoryStreamFixedSize &) = delete; + + bool isOpen() override { + return mBuffer != nullptr; + } + + bool isFile() override { + return false; + } + + bool isDataWrapper() override { + return true; + } + + std::string getPath() override { + return ""; + } + void *getData() override { + return mBuffer.get(); + } + + size_t getSize() override { + return mSize; + } + + // Write data to the memory stream + int64_t write(const void *data, const size_t size) override { + size_t copySize = size; + if (mPosition + size > mSize) { + copySize = mSize - mPosition; + } + if (copySize == 0) { + return 0; + } + std::memcpy(mBuffer.get() + mPosition, data, size); + mPosition += size; + return size; + } + + // Read data from the memory stream + int64_t read(void *output, const size_t size) override { + size_t readSize = size; + if (mPosition + size > mSize) { + readSize = mSize - mPosition; + } + if (readSize == 0) { + return 0; + } + std::memcpy(output, mBuffer.get() + mPosition, size); + mPosition += size; + return size; + } + + // Seek to a specific position + off_t seek(const int64_t offset, const int whence) override { + int64_t newPos = mPosition; + + if (whence == SEEK_SET) { + newPos = offset; + } else if (whence == SEEK_CUR) { + newPos += offset; + } else if (whence == SEEK_END) { + newPos = mSize + offset; + } + if (newPos < 0) { + mPosition = 0; + } else { + mPosition = newPos; + } + if (mPosition > mSize) { + mPosition = mSize; + } + + return mPosition; + } + + // Get the current position + [[nodiscard]] off_t tell() const override { + return mPosition; + } + + private: + std::unique_ptr mBuffer; + size_t mSize; + size_t mPosition; + }; + + class FileStream final : public ReadWriteStreamIF { + public: + explicit FileStream(const std::string_view path, const std::string_view mode) : mPath(path) { + mFile = ::fopen(path.data(), mode.data()); + if (mFile != nullptr) { + setvbuf(mFile, nullptr, _IOFBF, 128 * 1024); + ::fseek(mFile, 0, SEEK_END); + mSize = ::ftell(mFile); + ::fseek(mFile, 0, SEEK_SET); + mPosition = 0; + } + } + + ~FileStream() override { + releaseFile(); + } + + FileStream(const FileStream &) = delete; + FileStream &operator=(const FileStream &) = delete; + + FileStream(FileStream &&other) noexcept { + releaseFile(); + mFile = other.mFile; + mPath = std::move(other.mPath); + mSize = other.mSize; + mPosition = other.mPosition; + other.mFile = nullptr; + other.mSize = 0; + other.mPosition = 0; + } + + void releaseFile() { + if (mFile != nullptr) { + ::fclose(mFile); + mFile = nullptr; + } + } + + FileStream &operator=(FileStream &&other) noexcept { + if (this != &other) { + releaseFile(); + mFile = other.mFile; + mPath = std::move(other.mPath); + mSize = other.mSize; + mPosition = other.mPosition; + other.mFile = nullptr; + other.mSize = 0; + other.mPosition = 0; + } + return *this; + } + + bool isOpen() override { + return mFile != nullptr; + } + + bool isFile() override { + return true; + } + + bool isDataWrapper() override { + return false; + } + + std::string getPath() override { + return mPath; + } + + void *getData() override { + return nullptr; + } + + size_t getSize() override { + return mSize; + } + + // Write data to the memory stream + int64_t write(const void *data, const size_t size) override { + const auto res = ::fwrite(data, 1, size, mFile); + mPosition += res; + if (mPosition > mSize) { + mSize = mPosition; + } + return res; + } + + // Read data from the memory stream + int64_t read(void *output, const size_t size) override { + const auto res = ::fread(output, 1, size, mFile); + mPosition += res; + return res; + } + + // Seek to a specific position + off_t seek(const int64_t offset, const int whence) override { + const auto res = ::fseek(mFile, offset, whence); + mPosition = res; + return res; + } + + // Get the current position + [[nodiscard]] off_t tell() const override { + return mPosition; + } + + private: + FILE *mFile = nullptr; + std::string mPath; + size_t mSize = 0; + size_t mPosition = 0; + }; + + std::unique_ptr getNewFileOutputStream() { + std::unique_ptr dataStream; + if (USED_FILE_PATH != TEMP_FILE_1) { + if (dataStream = std::make_unique(TEMP_FILE_1, "w+"); dataStream->isOpen()) { + return dataStream; + } + } + if (USED_FILE_PATH != TEMP_FILE_2) { + if (dataStream = std::make_unique(TEMP_FILE_2, "w+"); dataStream->isOpen()) { + return dataStream; + } + } + if (USED_FILE_PATH != TEMP_FILE_3) { + if (dataStream = std::make_unique(TEMP_FILE_3, "w+"); dataStream->isOpen()) { + return dataStream; + } + } + if (USED_FILE_PATH != TEMP_FILE_4) { + if (dataStream = std::make_unique(TEMP_FILE_4, "w+"); dataStream->isOpen()) { + return dataStream; + } + } + return {}; + } + + bool saveDataToFile(const std::unique_ptr &inputStream, std::string &loadedPathOut, const NotificationModuleHandle notificationHandle) { + if (USED_FILE_PATH != TEMP_FILE_1 && FSUtils::saveBufferToFile(TEMP_FILE_1, inputStream->getData(), inputStream->getSize(), notificationHandle)) { + loadedPathOut = TEMP_FILE_1; + return true; + } + if (USED_FILE_PATH != TEMP_FILE_2 && FSUtils::saveBufferToFile(TEMP_FILE_2, inputStream->getData(), inputStream->getSize(), notificationHandle)) { + loadedPathOut = TEMP_FILE_2; + return true; + } + if (USED_FILE_PATH != TEMP_FILE_3 && FSUtils::saveBufferToFile(TEMP_FILE_3, inputStream->getData(), inputStream->getSize(), notificationHandle)) { + loadedPathOut = TEMP_FILE_3; + return true; + } + if (USED_FILE_PATH != TEMP_FILE_4 && FSUtils::saveBufferToFile(TEMP_FILE_4, inputStream->getData(), inputStream->getSize(), notificationHandle)) { + loadedPathOut = TEMP_FILE_4; + return true; + } + return false; + } + +} // namespace TcpReceiver::TcpReceiver(int32_t port) : CThread(CThread::eAttributeAffCore1, 16, 0x20000, nullptr, nullptr, "Wiiload Thread"), exitRequested(false), serverPort(port), serverSocket(-1) { @@ -94,6 +357,7 @@ void TcpReceiver::executeThread() { if (clientSocket >= 0) { uint32_t ipAddress = clientAddr.sin_addr.s_addr; DEBUG_FUNCTION_LINE("Waiting for wiiload connection"); + auto result = loadToMemory(clientSocket, ipAddress); close(clientSocket); @@ -127,6 +391,9 @@ void TcpReceiver::executeThread() { case NO_ACTIVE_ACCOUNT: NotificationModule_AddErrorNotification("Wiiload plugin: Failed to launch homebrew. No active account loaded."); break; + case STREAM_ERROR: + NotificationModule_AddErrorNotification("Wiiload plugin: Failed to launch homebrew. Invalid stream type"); + break; } if (result == SUCCESS) { @@ -145,19 +412,32 @@ void TcpReceiver::executeThread() { DEBUG_FUNCTION_LINE("Stopping wiiload server."); } -TcpReceiver::eLoadResults TcpReceiver::tryLoadWUHB(void *data, uint32_t fileSize, std::string &loadedPathOut) { - if (memcmp(data, "WUHB", 4) == 0) { +TcpReceiver::eLoadResults TcpReceiver::tryLoadWUHB(std::unique_ptr &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut) { + inputStream->seek(0, SEEK_SET); + std::vector header(4); + inputStream->read(header.data(), 4); + if (memcmp(header.data(), "WUHB", 4) == 0) { DEBUG_FUNCTION_LINE("Try to load a .wuhb"); - if (!FSUtils::CreateSubfolder(APPS_TEMP_PATH)) { - DEBUG_FUNCTION_LINE_WARN("Failed to create directory: %s", APPS_TEMP_PATH); - return FILE_SAVE_BUFFER_ERROR; - } - if (FSUtils::saveBufferToFile(WUHB_TEMP_FILE, data, fileSize)) { - loadedPathOut = WUHB_TEMP_FILE_EX; - } else if (FSUtils::saveBufferToFile(WUHB_TEMP_FILE_2, data, fileSize)) { - loadedPathOut = WUHB_TEMP_FILE_2_EX; + if (inputStream->isDataWrapper() && inputStream->getData()) { + if (!FSUtils::CreateSubfolder(APPS_TEMP_PATH)) { + DEBUG_FUNCTION_LINE_WARN("Failed to create directory: %s", APPS_TEMP_PATH); + return FILE_SAVE_BUFFER_ERROR; + } + if (!saveDataToFile(inputStream, loadedPathOut, notificationHandle)) { + DEBUG_FUNCTION_LINE_WARN("Failed to save .wuhb file to the sd card. Launching will be aborted."); + return FILE_SAVE_BUFFER_ERROR; + } + } else if (inputStream->isFile()) { + loadedPathOut = inputStream->getPath(); + + if (loadedPathOut.empty()) { + DEBUG_FUNCTION_LINE_WARN("Path was empty. Launching will be aborted."); + return FILE_SAVE_BUFFER_ERROR; + } + // Release stream so we can access it! + inputStream.reset(); } else { - DEBUG_FUNCTION_LINE_WARN("Failed to save .wuhb file to the sd card. Launching will be aborted."); + DEBUG_FUNCTION_LINE_WARN("Invalid stream type detected. Launching will be aborted."); return FILE_SAVE_BUFFER_ERROR; } return SUCCESS; @@ -166,17 +446,32 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadWUHB(void *data, uint32_t fileSize return UNSUPPORTED_FORMAT; } -TcpReceiver::eLoadResults TcpReceiver::tryLoadRPX(uint8_t *data, uint32_t fileSize, std::string &loadedPathOut) { - if (data[0x7] == 0xCA && data[0x8] == 0xFE && data[0x9] != 0x50 && data[0xA] != 0x4C) { +TcpReceiver::eLoadResults TcpReceiver::tryLoadRPX(std::unique_ptr &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut) { + inputStream->seek(0, SEEK_SET); + std::vector header(16); + inputStream->read(header.data(), 16); + if (header[0x7] == 0xCA && header[0x8] == 0xFE && header[0x9] != 0x50 && header[0xA] != 0x4C) { DEBUG_FUNCTION_LINE("Try to load a .rpx"); - if (!FSUtils::CreateSubfolder(APPS_TEMP_PATH)) { - DEBUG_FUNCTION_LINE_WARN("Failed to create directory: %s", APPS_TEMP_PATH); - return FILE_SAVE_BUFFER_ERROR; - } - if (FSUtils::saveBufferToFile(RPX_TEMP_FILE, data, fileSize)) { - loadedPathOut = RPX_TEMP_FILE_EX; + if (inputStream->isDataWrapper() && inputStream->getData()) { + if (!FSUtils::CreateSubfolder(APPS_TEMP_PATH)) { + DEBUG_FUNCTION_LINE_WARN("Failed to create directory: %s", APPS_TEMP_PATH); + return FILE_SAVE_BUFFER_ERROR; + } + if (!saveDataToFile(inputStream, loadedPathOut, notificationHandle)) { + DEBUG_FUNCTION_LINE_WARN("Failed to save .rpx file to the sd card. Launching will be aborted."); + return FILE_SAVE_BUFFER_ERROR; + } + } else if (inputStream->isFile()) { + loadedPathOut = inputStream->getPath(); + + if (loadedPathOut.empty()) { + DEBUG_FUNCTION_LINE_WARN("Path was empty. Launching will be aborted."); + return FILE_SAVE_BUFFER_ERROR; + } + // Release stream so we can access it! + inputStream.reset(); } else { - DEBUG_FUNCTION_LINE_WARN("Failed to save .rpx file to the sd card. Launching will be aborted."); + DEBUG_FUNCTION_LINE_WARN("Invalid stream type detected. Launching will be aborted."); return FILE_SAVE_BUFFER_ERROR; } return SUCCESS; @@ -185,11 +480,32 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadRPX(uint8_t *data, uint32_t fileSi return UNSUPPORTED_FORMAT; } -TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSize) { - if (data[0x7] == 0xCA && data[0x8] == 0xFE && data[0x9] == 0x50 && data[0xA] == 0x4C) { +TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(std::unique_ptr &inputStream, NotificationModuleHandle notificationHandle) { + inputStream->seek(0, SEEK_SET); + std::vector header(16); + inputStream->read(header.data(), 16); + if (header[0x7] == 0xCA && header[0x8] == 0xFE && header[0x9] == 0x50 && header[0xA] == 0x4C) { PluginBackendApiErrorType err; PluginBackendPluginParseError parseErr; - auto newContainerOpt = WUPSBackend::PluginUtils::getPluginForBuffer((char *) data, fileSize, err, parseErr); + std::optional newContainerOpt; + if (inputStream->isFile()) { + const auto path = inputStream->getPath(); + // Reset stream ptr so we can access the file! + inputStream.reset(); + + newContainerOpt = WUPSBackend::PluginUtils::getPluginForPath(path, err, parseErr); + // We can delete file no from the sd card + remove(path.c_str()); + } else if (inputStream->isDataWrapper()) { + DEBUG_FUNCTION_LINE_ERR("Loading from buffer"); + newContainerOpt = WUPSBackend::PluginUtils::getPluginForBuffer(static_cast(inputStream->getData()), inputStream->getSize(), err, parseErr); + // release the memory as early as possible. + inputStream.reset(); + } else { + DEBUG_FUNCTION_LINE_ERR("Invalid stream type. This should never happen!"); + return STREAM_ERROR; + } + if (newContainerOpt) { auto pluginList = WUPSBackend::PluginUtils::getLoadedPlugins(err); if (err != PLUGIN_BACKEND_API_ERROR_NONE) { @@ -199,13 +515,12 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSi const auto &metaInformation = newContainerOpt->getMetaInformation(); // remove plugins with the same name and author as our new plugin - pluginList.erase(std::remove_if(pluginList.begin(), pluginList.end(), - [&metaInformation](auto &plugin) { - auto res = plugin.getMetaInformation().getName() == metaInformation.getName() && - plugin.getMetaInformation().getAuthor() == metaInformation.getAuthor(); - return res; - }), - pluginList.end()); + std::erase_if(pluginList, + [&metaInformation](auto &plugin) { + auto res = plugin.getMetaInformation().getName() == metaInformation.getName() && + plugin.getMetaInformation().getAuthor() == metaInformation.getAuthor(); + return res; + }); // add the new plugin pluginList.push_back(std::move(newContainerOpt.value())); #ifdef VERBOSE_DEBUG @@ -222,7 +537,7 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSi } return SUCCESS; } else { - DEBUG_FUNCTION_LINE_ERR("Failed to parse plugin for buffer: %08X size %d. Error: %s", data, fileSize, WUPSBackend::GetStatusStr(err)); + DEBUG_FUNCTION_LINE_ERR("Failed to parse plugin. Error: %s", WUPSBackend::GetStatusStr(err)); return PLUGIN_PARSE_FAILED; } } @@ -230,14 +545,23 @@ TcpReceiver::eLoadResults TcpReceiver::tryLoadWPS(uint8_t *data, uint32_t fileSi return UNSUPPORTED_FORMAT; } -TcpReceiver::eLoadResults TcpReceiver::loadBinary(void *data, uint32_t fileSize) { +TcpReceiver::eLoadResults TcpReceiver::loadBinary(std::unique_ptr &&inputStream, const NotificationModuleHandle notificationHandle) { std::string loadedPath; - eLoadResults error; - if ((error = tryLoadWUHB(data, fileSize, loadedPath)) != UNSUPPORTED_FORMAT || (error = tryLoadRPX((uint8_t *) data, fileSize, loadedPath)) != UNSUPPORTED_FORMAT) { + if (eLoadResults error; (error = tryLoadWUHB(inputStream, notificationHandle, loadedPath)) != UNSUPPORTED_FORMAT || (error = tryLoadRPX(inputStream, notificationHandle, loadedPath)) != UNSUPPORTED_FORMAT) { if (error != SUCCESS) { return error; } + // lets save which file we don't want to override + USED_FILE_PATH = loadedPath; + + const auto index = loadedPath.find("fs:/vol/external01/"); + if (index != std::string::npos) { + loadedPath.erase(index, strlen("fs:/vol/external01/")); + } + + DEBUG_FUNCTION_LINE_VERBOSE("Loaded file: other %s", loadedPath.c_str()); + nn::act::Initialize(); bool accountLoaded = nn::act::IsSlotOccupied(nn::act::GetSlotNo()); nn::act::Finalize(); @@ -246,123 +570,173 @@ TcpReceiver::eLoadResults TcpReceiver::loadBinary(void *data, uint32_t fileSize) return NO_ACTIVE_ACCOUNT; } - RPXLoaderStatus launchRes; - if ((launchRes = RPXLoader_LaunchHomebrew(loadedPath.c_str())) != RPX_LOADER_RESULT_SUCCESS) { - DEBUG_FUNCTION_LINE_ERR("Failed to start %s %s", loadedPath.c_str(), RPXLoader_GetStatusStr(launchRes)); + if (const RPXLoaderStatus launchRes = RPXLoader_LaunchHomebrew(loadedPath.c_str()); launchRes != RPX_LOADER_RESULT_SUCCESS) { + DEBUG_FUNCTION_LINE_ERR("Failed to start \"%s\": %s", loadedPath.c_str(), RPXLoader_GetStatusStr(launchRes)); return LAUNCH_FAILED; } - } else if ((error = tryLoadWPS((uint8_t *) data, fileSize)) != UNSUPPORTED_FORMAT) { + + NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Launching homebrew..."); + return SUCCESS; + } else if ((error = tryLoadWPS(inputStream, notificationHandle)) != UNSUPPORTED_FORMAT) { if (error != SUCCESS) { return error; } _SYSLaunchTitleWithStdArgsInNoSplash(OSGetTitleID(), nullptr); - } else { - return UNSUPPORTED_FORMAT; + NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Reloading with new plugin..."); + return SUCCESS; } - return SUCCESS; + return UNSUPPORTED_FORMAT; } -std::unique_ptr TcpReceiver::uncompressData(uint32_t fileSize, uint32_t fileSizeUnc, std::unique_ptr &&inData, uint32_t &fileSizeOut, eLoadResults &err) { - std::unique_ptr inflatedData; - uint8_t *in_data_raw = inData.get(); - // We need to unzip... - if (in_data_raw[0] == 'P' && in_data_raw[1] == 'K' && in_data_raw[2] == 0x03 && in_data_raw[3] == 0x04) { - // Section is compressed, inflate - inflatedData = make_unique_nothrow(fileSizeUnc); - if (!inflatedData) { - DEBUG_FUNCTION_LINE_ERR("malloc failed"); - err = NOT_ENOUGH_MEMORY; - return {}; - } +constexpr size_t CHUNK_SIZE = 128 * 1024; // Size of chunks to process - int32_t ret; - z_stream s = {}; - - s.zalloc = Z_NULL; - s.zfree = Z_NULL; - s.opaque = Z_NULL; - - ret = inflateInit(&s); - if (ret != Z_OK) { - DEBUG_FUNCTION_LINE_ERR("inflateInit failed %i", ret); - err = FILE_UNCOMPRESS_ERROR; - return {}; - } - - s.avail_in = fileSize; - s.next_in = (Bytef *) inflatedData.get(); - - s.avail_out = fileSizeUnc; - s.next_out = (Bytef *) inflatedData.get(); - - ret = inflate(&s, Z_FINISH); - if (ret != Z_OK && ret != Z_STREAM_END) { - DEBUG_FUNCTION_LINE_ERR("inflate failed %i", ret); - err = FILE_UNCOMPRESS_ERROR; - return {}; - } - - inflateEnd(&s); - } else { - // Section is compressed, inflate - inflatedData = make_unique_nothrow(fileSizeUnc); - if (!inflatedData) { - DEBUG_FUNCTION_LINE_ERR("malloc failed"); - err = NOT_ENOUGH_MEMORY; - return {}; - } - - uLongf f = fileSizeUnc; - int32_t result = uncompress((Bytef *) inflatedData.get(), &f, (Bytef *) in_data_raw, fileSize); - if (result != Z_OK) { - DEBUG_FUNCTION_LINE_ERR("uncompress failed %i", result); - err = FILE_UNCOMPRESS_ERROR; - return {}; - } - - fileSizeUnc = f; +std::unique_ptr TcpReceiver::uncompressData(uint32_t fileSizeUnc, std::unique_ptr &&inputStream, const NotificationModuleHandle notificationHandle, const bool toFile, eLoadResults &err) { + if (!inputStream->isOpen()) { + DEBUG_FUNCTION_LINE_ERR("inputStream is not open"); + return {}; } - - fileSizeOut = fileSizeUnc; - err = SUCCESS; - return inflatedData; -} - -std::unique_ptr TcpReceiver::receiveData(int32_t clientSocket, uint32_t fileSize, eLoadResults &err) { - uint32_t bytesRead = 0; - auto dataOut = make_unique_nothrow(fileSize); - if (!dataOut) { - err = NOT_ENOUGH_MEMORY; + std::unique_ptr outputStream = {}; + if (toFile) { + outputStream = getNewFileOutputStream(); + } else { + outputStream = std::make_unique(fileSizeUnc); + } + if (!outputStream || !outputStream->isOpen()) { + DEBUG_FUNCTION_LINE_ERR("Failed to open or create outputStream"); return {}; } + inputStream->seek(0, SEEK_SET); + outputStream->seek(0, SEEK_SET); + + std::vector inBuffer(CHUNK_SIZE); + std::vector outBuffer(CHUNK_SIZE); + + z_stream strm = {}; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + // Initialize the inflate state + if (inflateInit(&strm) != Z_OK) { + DEBUG_FUNCTION_LINE_ERR("Failed to call inflateInit"); + return {}; + } + + int ret = Z_STREAM_ERROR; + do { + // Read a chunk of compressed data + auto readRes = inputStream->read(inBuffer.data(), CHUNK_SIZE); + + std::string progressStr = string_format("[Wiiload] Decompressing data... %.2f%%", static_cast(inputStream->tell()) / static_cast(inputStream->getSize()) * 100.0f); + NotificationModule_UpdateDynamicNotificationText(notificationHandle, progressStr.c_str()); + + if (readRes <= 0) { + break; + } + + strm.avail_in = readRes; + strm.next_in = inBuffer.data(); + + // Decompress in a loop + do { + strm.avail_out = CHUNK_SIZE; + strm.next_out = outBuffer.data(); + + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: { + DEBUG_FUNCTION_LINE_ERR("inflate() failed"); + (void) inflateEnd(&strm); + return {}; + } + } + + auto have = CHUNK_SIZE - strm.avail_out; + if (outputStream->write(outBuffer.data(), have) != have) { + DEBUG_FUNCTION_LINE_ERR("write() failed"); + (void) inflateEnd(&strm); + return {}; + } + } while (strm.avail_out == 0); + + } while (ret != Z_STREAM_END); + + // clean up uncompressed file + if (inputStream->isFile()) { + const auto path = inputStream->getPath(); + inputStream.reset(); + remove(path.c_str()); + } + + // Clean up + inflateEnd(&strm); + if (ret != Z_STREAM_END) { + DEBUG_FUNCTION_LINE_ERR("Failed to uncompress data"); + NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Decompressing data... failed"); + return {}; + } + + NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Decompressing data... 100.00%"); + return outputStream; +} + +bool TcpReceiver::receiveData(const int32_t clientSocket, const std::unique_ptr &output, const uint32_t fileSize, const NotificationModuleHandle notificationHandle, eLoadResults &err) { + uint32_t bytesRead = 0; + std::vector inBuffer(CHUNK_SIZE); // Copy binary in memory while (bytesRead < fileSize) { - uint32_t blockSize = 0x1000; + std::string progressStr = string_format("[Wiiload] Receiving data... %.2f%%", static_cast(bytesRead) / static_cast(fileSize) * 100.0f); + NotificationModule_UpdateDynamicNotificationText(notificationHandle, progressStr.c_str()); + + uint32_t blockSize = CHUNK_SIZE; if (blockSize > (fileSize - bytesRead)) blockSize = fileSize - bytesRead; - int32_t ret = recv(clientSocket, dataOut.get() + bytesRead, blockSize, 0); + const int32_t ret = recv(clientSocket, inBuffer.data(), blockSize, 0); if (ret <= 0) { DEBUG_FUNCTION_LINE_ERR("Failed to receive file"); break; } + int64_t writtenData = 0; + while (writtenData < ret) { + const int64_t toWrite = ret - writtenData; + const auto res = output->write(inBuffer.data() + writtenData, toWrite); + if (res <= 0) { + if (res < 0) { + DEBUG_FUNCTION_LINE_ERR("Failed to receive file"); + } + break; + } + writtenData += res; + } + if (writtenData != ret) { + DEBUG_FUNCTION_LINE_ERR("Failed to write to file"); + break; + } bytesRead += ret; } if (bytesRead != fileSize) { + NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Receiving data... failed"); DEBUG_FUNCTION_LINE_ERR("File loading not finished, %i of %i bytes received", bytesRead, fileSize); err = RECV_ERROR; - return {}; + return false; } + + NotificationModule_UpdateDynamicNotificationText(notificationHandle, "[Wiiload] Receiving data... 100.00%"); + err = SUCCESS; - return dataOut; + return true; } TcpReceiver::eLoadResults TcpReceiver::loadToMemory(int32_t clientSocket, uint32_t ipAddress) { DEBUG_FUNCTION_LINE("Loading file from ip %08X", ipAddress); - uint32_t fileSize = 0; uint32_t fileSizeUnc = 0; uint8_t haxx[8] = {}; @@ -379,16 +753,38 @@ TcpReceiver::eLoadResults TcpReceiver::loadToMemory(int32_t clientSocket, uint32 return RECV_ERROR; } } - TcpReceiver::eLoadResults err = UNSUPPORTED_FORMAT; - auto receivedData = receiveData(clientSocket, fileSize, err); - if (err != SUCCESS) { + eLoadResults err = UNSUPPORTED_FORMAT; + + NotificationModuleHandle notificationHandle; + if (NotificationModule_AddDynamicNotification("[Wiiload] Receiving binary via network", ¬ificationHandle) != NOTIFICATION_MODULE_RESULT_SUCCESS) { + notificationHandle = 0; + } + + const bool asFile = fileSize > 1; + std::unique_ptr dataStream; + if (asFile) { + dataStream = getNewFileOutputStream(); + } else { + dataStream = std::make_unique(fileSize); + } + + if (!dataStream || !receiveData(clientSocket, dataStream, fileSize, notificationHandle, err) || err != SUCCESS) { + DEBUG_FUNCTION_LINE_ERR("ERROR: Failed to receive file"); + NotificationModule_FinishDynamicNotification(notificationHandle, 0.5f); return err; - } else if (compressedData) { - receivedData = uncompressData(fileSize, fileSizeUnc, std::move(receivedData), fileSize, err); - if (!receivedData || err != SUCCESS) { + } + + if (compressedData) { + dataStream = uncompressData(fileSizeUnc, std::move(dataStream), notificationHandle, asFile, err); + if (!dataStream || err != SUCCESS) { + DEBUG_FUNCTION_LINE_ERR("ERROR: Failed to uncompress file"); + NotificationModule_FinishDynamicNotification(notificationHandle, 0.5f); return err; } } + dataStream->seek(0, SEEK_SET); - return loadBinary(receivedData.get(), fileSize); + const auto res = loadBinary(std::move(dataStream), notificationHandle); + NotificationModule_FinishDynamicNotification(notificationHandle, 0.5f); + return res; } diff --git a/src/utils/TcpReceiver.h b/src/utils/TcpReceiver.h index a1c5b5d..64715e6 100644 --- a/src/utils/TcpReceiver.h +++ b/src/utils/TcpReceiver.h @@ -1,10 +1,13 @@ #pragma once +#include "CThread.h" + +#include + #include #include -#include -#include "CThread.h" +class ReadWriteStreamIF; class TcpReceiver : public CThread { public: @@ -19,6 +22,7 @@ public: RECV_ERROR = -7, LAUNCH_FAILED = -8, NO_ACTIVE_ACCOUNT = -9, + STREAM_ERROR = -10, }; explicit TcpReceiver(int32_t port); @@ -33,12 +37,12 @@ private: static eLoadResults loadToMemory(int32_t clientSocket, uint32_t ipAddress); - static TcpReceiver::eLoadResults tryLoadWUHB(void *data, uint32_t fileSize, std::string &loadedPathOut); - static TcpReceiver::eLoadResults tryLoadRPX(uint8_t *data, uint32_t fileSize, std::string &loadedPathOut); - static TcpReceiver::eLoadResults tryLoadWPS(uint8_t *data, uint32_t fileSize); - static TcpReceiver::eLoadResults loadBinary(void *data, uint32_t fileSize); - static std::unique_ptr receiveData(int32_t clientSocket, uint32_t fileSize, eLoadResults &err); - static std::unique_ptr uncompressData(uint32_t fileSize, uint32_t fileSizeUnc, std::unique_ptr &&in_out_data, uint32_t &fileSizeOut, eLoadResults &err); + static TcpReceiver::eLoadResults tryLoadWUHB(std::unique_ptr &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut); + static TcpReceiver::eLoadResults tryLoadRPX(std::unique_ptr &inputStream, NotificationModuleHandle notificationHandle, std::string &loadedPathOut); + static TcpReceiver::eLoadResults tryLoadWPS(std::unique_ptr &inputStream, NotificationModuleHandle notificationHandle); + static TcpReceiver::eLoadResults loadBinary(std::unique_ptr &&inputStream, NotificationModuleHandle notificationHandle); + static bool receiveData(int32_t clientSocket, const std::unique_ptr &output, uint32_t fileSize, NotificationModuleHandle notificationHandle, eLoadResults &err); + static std::unique_ptr uncompressData(uint32_t fileSizeUnc, std::unique_ptr &&inputStream, NotificationModuleHandle notificationHandle, bool toFile, eLoadResults &err); bool exitRequested; int32_t serverPort; diff --git a/src/utils/net.c b/src/utils/net.cpp similarity index 55% rename from src/utils/net.c rename to src/utils/net.cpp index 98740d9..c813115 100644 --- a/src/utils/net.c +++ b/src/utils/net.cpp @@ -1,28 +1,25 @@ #include "net.h" + +#include + #include -static volatile int socket_lock __attribute__((section(".data"))) = 0; +static std::mutex sSocketMutex; -int32_t recvwait(int32_t sock, void *buffer, int32_t len) { - while (socket_lock) { - usleep(1000); - } - socket_lock = 1; - int32_t ret; +int32_t recvwait(const int32_t sock, void *buffer, int32_t len) { + std::lock_guard lock(sSocketMutex); while (len > 0) { - ret = recv(sock, buffer, len, 0); + const int32_t ret = recv(sock, buffer, len, 0); if (ret < 0) { - socket_lock = 0; return ret; } len -= ret; buffer = (void *) (((char *) buffer) + ret); } - socket_lock = 0; return 0; } -uint8_t recvbyte(int32_t sock) { +uint8_t recvbyte(const int32_t sock) { unsigned char buffer[1]; int32_t ret; @@ -32,7 +29,7 @@ uint8_t recvbyte(int32_t sock) { return buffer[0]; } -uint32_t recvword(int32_t sock) { +uint32_t recvword(const int32_t sock) { uint32_t result; int32_t ret; @@ -42,16 +39,11 @@ uint32_t recvword(int32_t sock) { return result; } -int32_t checkbyte(int32_t sock) { - while (socket_lock) { - usleep(1000); - } - socket_lock = 1; +int32_t checkbyte(const int32_t sock) { + std::lock_guard lock(sSocketMutex); unsigned char buffer[1]; - int32_t ret; - ret = recv(sock, buffer, 1, MSG_DONTWAIT); - socket_lock = 0; + const int32_t ret = recv(sock, buffer, 1, MSG_DONTWAIT); if (ret < 0) return ret; if (ret == 0) @@ -60,23 +52,17 @@ int32_t checkbyte(int32_t sock) { } int32_t sendwait(int32_t sock, const void *buffer, int32_t len) { - while (socket_lock) { - usleep(1000); - } - socket_lock = 1; - int32_t ret; + std::lock_guard lock(sSocketMutex); while (len > 0) { - // For some reason the send blocks/crashes if the buffer is too big.. - int cur_length = len <= 0x30 ? len : 0x30; - ret = send(sock, buffer, cur_length, 0); + // For some reason the send blocks/crashes if the buffer is too big... + const int cur_length = len <= 0x30 ? len : 0x30; + const int32_t ret = send(sock, buffer, cur_length, 0); if (ret < 0) { - socket_lock = 0; return ret; } len -= ret; buffer = (void *) (((char *) buffer) + ret); } - socket_lock = 0; return 0; } diff --git a/src/utils/utils.h b/src/utils/utils.h index e48e53a..06f80ac 100644 --- a/src/utils/utils.h +++ b/src/utils/utils.h @@ -3,6 +3,7 @@ #include #include + template std::unique_ptr make_unique_nothrow(Args &&...args) noexcept(noexcept(T(std::forward(args)...))) { return std::unique_ptr(new (std::nothrow) T(std::forward(args)...)); @@ -11,4 +12,18 @@ std::unique_ptr make_unique_nothrow(Args &&...args) noexcept(noexcept(T(std:: template inline typename std::unique_ptr make_unique_nothrow(size_t num) noexcept { return std::unique_ptr(new (std::nothrow) std::remove_extent_t[num]()); -} \ No newline at end of file +} + +template +std::string string_format(const std::string_view format, Args... args) { + const int size_s = std::snprintf(nullptr, 0, format.data(), args...) + 1; // Extra space for '\0' + const auto size = static_cast(size_s); + const auto buf = make_unique_nothrow(size); + if (!buf) { + DEBUG_FUNCTION_LINE_ERR("string_format failed, not enough memory"); + OSFatal("string_format failed, not enough memory"); + return std::string(""); + } + std::snprintf(buf.get(), size, format.data(), args...); + return std::string(buf.get(), buf.get() + size - 1); // We don't want the '\0' inside +}