Merge branch 'splitfiles'

include/downloader.h

@@ -68,19 +68,28 @@ struct ChunkMemoryStruct
 
 typedef struct
 {
-	std::string filepath;
-	off_t comp_size;
-	off_t uncomp_size;
-	off_t start_offset_zip;
-	off_t start_offset_mojosetup;
-	off_t end_offset;
-	uint16_t file_attributes;
-	uint32_t crc32;
-	time_t timestamp;
+    std::string filepath;
+    off_t comp_size;
+    off_t uncomp_size;
+    off_t start_offset_zip;
+    off_t start_offset_mojosetup;
+    off_t end_offset;
+    uint16_t file_attributes;
+    uint32_t crc32;
+    time_t timestamp;
 
-	std::string installer_url;
+    std::string installer_url;
+
+    // For split file handling
+    bool isSplitFile = false;
+    std::string splitFileBasePath;
+    std::string splitFilePartExt;
+    off_t splitFileStartOffset;
+    off_t splitFileEndOffset;
 } zipFileEntry;
 
+typedef std::map<std::string,std::vector<zipFileEntry>> splitFilesMap;
+
 class Downloader
 {
     public:
@@ -140,6 +149,7 @@ class Downloader
         std::vector<std::string> galaxyGetOrphanedFiles(const std::vector<galaxyDepotItem>& items, const std::string& install_path);
         static void processGalaxyDownloadQueue(const std::string& install_path, Config conf, const unsigned int& tid);
         void galaxyInstallGame_MojoSetupHack(const std::string& product_id);
+        void galaxyInstallGame_MojoSetupHack_CombineSplitFiles(const splitFilesMap& mSplitFiles, const bool& bAppendtoFirst = false);
         static void processGalaxyDownloadQueue_MojoSetupHack(Config conf, const unsigned int& tid);
         int mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEntry>& vFiles);
         std::string getGalaxyInstallDirectory(galaxyAPI *galaxyHandle, const Json::Value& manifest);

include/util.h

@@ -54,6 +54,7 @@ namespace Util
     std::string makeFilepath(const std::string& directory, const std::string& path, const std::string& gamename, std::string subdirectory = "", const unsigned int& platformId = 0, const std::string& dlcname = "");
     std::string makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory = "");
     std::string getFileHash(const std::string& filename, unsigned hash_id);
+    std::string getFileHashRange(const std::string& filepath, unsigned hash_id, off_t range_start = 0, off_t range_end = 0);
     std::string getChunkHash(unsigned char* chunk, uintmax_t chunk_size, unsigned hash_id);
     int createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir = std::string());
     int getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf, std::string directory = std::string());

src/downloader.cpp

@@ -4192,6 +4192,50 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
         std::vector<zipFileEntry> vZipDirectories;
         std::vector<zipFileEntry> vZipFiles;
         std::vector<zipFileEntry> vZipFilesSymlink;
+        std::vector<zipFileEntry> vZipFilesSplit;
+
+        // Determine if installer contains split files and get list of base file paths
+        std::vector<std::string> vSplitFileBasePaths;
+        for (const auto& zfe : zipFileEntries)
+        {
+            std::string noarch = "data/noarch/";
+            std::string split_files = noarch + "support/split_files";
+            if (zfe.filepath.find(split_files) != std::string::npos)
+            {
+                std::cout << "Getting info about split files" << std::endl;
+                std::string url = zfe.installer_url;
+                std::string dlrange = std::to_string(zfe.start_offset_mojosetup) + "-" + std::to_string(zfe.end_offset);
+                curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
+
+                std::stringstream splitfiles_compressed;
+                std::stringstream splitfiles_uncompressed;
+
+                CURLcode result = CURLE_RECV_ERROR;
+                curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, writeMemoryCallback);
+                curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &splitfiles_compressed);
+                curl_easy_setopt(curlhandle, CURLOPT_RANGE, dlrange.c_str());
+                result = curl_easy_perform(curlhandle);
+                curl_easy_setopt(curlhandle, CURLOPT_RANGE, NULL);
+
+                if (result == CURLE_OK)
+                {
+                    if (ZipUtil::extractStream(&splitfiles_compressed, &splitfiles_uncompressed) == 0)
+                    {
+                        std::string path;
+                        while (std::getline(splitfiles_uncompressed, path))
+                        {
+                            // Replace the leading "./" in base file path with install path
+                            Util::replaceString(path, "./", install_path);
+                            while (Util::replaceString(path, "//", "/")); // Replace any double slashes with single slash
+                            vSplitFileBasePaths.push_back(path);
+                        }
+                    }
+                }
+            }
+        }
+
+        bool bContainsSplitFiles = !vSplitFileBasePaths.empty();
+
         for (std::uintmax_t i = 0; i < zipFileEntries.size(); ++i)
         {
             // Ignore all files and directories that are not in "data/noarch/" directory
@@ -4208,7 +4252,45 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
             else if (ZipUtil::isSymlink(zfe.file_attributes))
                 vZipFilesSymlink.push_back(zfe);
             else
-                vZipFiles.push_back(zfe);
+            {
+                // Check for split files
+                if (bContainsSplitFiles)
+                {
+                    boost::regex expression("^(.*)(\\.split\\d+)$");
+                    boost::match_results<std::string::const_iterator> what;
+                    if (boost::regex_search(zfe.filepath, what, expression))
+                    {
+                        std::string basePath = what[1];
+                        std::string partExt = what[2];
+
+                        // Check against list of base file paths read from "data/noarch/support/split_files"
+                        if (
+                            std::any_of(
+                                vSplitFileBasePaths.begin(),
+                                vSplitFileBasePaths.end(),
+                                [basePath](const std::string& path)
+                                {
+                                    return path == basePath;
+                                }
+                            )
+                        )
+                        {
+                            zfe.isSplitFile = true;
+                            zfe.splitFileBasePath = basePath;
+                            zfe.splitFilePartExt = partExt;
+                        }
+                    }
+
+                    if (zfe.isSplitFile)
+                        vZipFilesSplit.push_back(zfe);
+                    else
+                        vZipFiles.push_back(zfe);
+                }
+                else
+                {
+                     vZipFiles.push_back(zfe);
+                }
+            }
         }
 
         // Create directories
@@ -4224,6 +4306,42 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
             }
         }
 
+        // Set start and end offsets for split files
+        // Create map of split files for combining them later
+        splitFilesMap mSplitFiles;
+        if (!vZipFilesSplit.empty())
+        {
+            std::sort(vZipFilesSplit.begin(), vZipFilesSplit.end(), [](const zipFileEntry& i, const zipFileEntry& j) -> bool { return i.filepath < j.filepath; });
+
+            std::string prevBasePath = "";
+            off_t prevEndOffset = 0;
+            for (auto& zfe : vZipFilesSplit)
+            {
+                if (zfe.splitFileBasePath == prevBasePath)
+                    zfe.splitFileStartOffset = prevEndOffset;
+                else
+                    zfe.splitFileStartOffset = 0;
+
+                zfe.splitFileEndOffset = zfe.splitFileStartOffset + zfe.uncomp_size;
+
+                prevBasePath = zfe.splitFileBasePath;
+                prevEndOffset = zfe.splitFileEndOffset;
+
+                if (mSplitFiles.count(zfe.splitFileBasePath) > 0)
+                {
+                    mSplitFiles[zfe.splitFileBasePath].push_back(zfe);
+                }
+                else
+                {
+                    std::vector<zipFileEntry> vec;
+                    vec.push_back(zfe);
+                    mSplitFiles[zfe.splitFileBasePath] = vec;
+                }
+            }
+
+            vZipFiles.insert(std::end(vZipFiles), std::begin(vZipFilesSplit), std::end(vZipFilesSplit));
+        }
+
         // Add files to download queue
         for (std::uintmax_t i = 0; i < vZipFiles.size(); ++i)
         {
@@ -4259,6 +4377,12 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
 
         vThreads.clear();
         vDownloadInfo.clear();
+
+        // Combine split files
+        if (!mSplitFiles.empty())
+        {
+            this->galaxyInstallGame_MojoSetupHack_CombineSplitFiles(mSplitFiles, true);
+        }
     }
     else
     {
@@ -4266,6 +4390,126 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
     }
 }
 
+void Downloader::galaxyInstallGame_MojoSetupHack_CombineSplitFiles(const splitFilesMap& mSplitFiles, const bool& bAppendToFirst)
+{
+    for (const auto& baseFile : mSplitFiles)
+    {
+        // Check that all parts exist
+        bool bAllPartsExist = true;
+        for (const auto& splitFile : baseFile.second)
+        {
+            if (!boost::filesystem::exists(splitFile.filepath))
+            {
+                bAllPartsExist = false;
+                break;
+            }
+        }
+
+        bool bBaseFileExists = boost::filesystem::exists(baseFile.first);
+
+        if (!bAllPartsExist)
+        {
+            if (bBaseFileExists)
+            {
+                // Base file exist and we're missing parts.
+                // This should mean that we already have complete file.
+                // So we can safely skip this file without informing the user
+                continue;
+            }
+            else
+            {
+                // Base file doesn't exist and we're missing parts. Print message about it before skipping file.
+                std::cout << baseFile.first << " is missing parts. Skipping this file." << std::endl;
+                continue;
+            }
+        }
+
+        // Delete base file if it already exists
+        if (bBaseFileExists)
+        {
+            std::cout << baseFile.first << " already exists. Deleting old file." << std::endl;
+            if (!boost::filesystem::remove(baseFile.first))
+            {
+                std::cout << baseFile.first << ": Failed to delete" << std::endl;
+                continue;
+            }
+        }
+
+        std::cout << "Beginning to combine " << baseFile.first << std::endl;
+        std::ofstream ofs;
+
+        // Create base file for appending if we aren't appending to first part
+        if (!bAppendToFirst)
+        {
+            ofs.open(baseFile.first, std::ios_base::binary | std::ios_base::app);
+            if (!ofs.is_open())
+            {
+                std::cout << "Failed to create " << baseFile.first << std::endl;
+                continue;
+            }
+        }
+
+        for (const auto& splitFile : baseFile.second)
+        {
+            std::cout << "\t" << splitFile.filepath << std::endl;
+
+            // Append to first file is set and current file is first in vector.
+            // Open file for appending and continue to next file
+            if (bAppendToFirst && (&splitFile == &baseFile.second.front()))
+            {
+                ofs.open(splitFile.filepath, std::ios_base::binary | std::ios_base::app);
+                if (!ofs.is_open())
+                {
+                    std::cout << "Failed to open " << splitFile.filepath << std::endl;
+                    break;
+                }
+                continue;
+            }
+
+            std::ifstream ifs(splitFile.filepath, std::ios_base::binary);
+            if (!ifs)
+            {
+                std::cout << "Failed to open " << splitFile.filepath << ". Deleting incomplete file." << std::endl;
+
+                ofs.close();
+                if (!boost::filesystem::remove(baseFile.first))
+                {
+                    std::cout << baseFile.first << ": Failed to delete" << std::endl;
+                }
+                break;
+            }
+
+            ofs << ifs.rdbuf();
+            ifs.close();
+
+            // Delete split file
+            if (!boost::filesystem::remove(splitFile.filepath))
+            {
+                std::cout << splitFile.filepath << ": Failed to delete" << std::endl;
+            }
+        }
+
+        if (ofs)
+            ofs.close();
+
+        // Appending to first file so we must rename it
+        if (bAppendToFirst)
+        {
+            boost::filesystem::path splitFilePath = baseFile.second.front().filepath;
+            boost::filesystem::path baseFilePath = baseFile.first;
+
+            boost::system::error_code ec;
+            boost::filesystem::rename(splitFilePath, baseFilePath, ec);
+            if (ec)
+            {
+                std::cout << "Failed to rename " << splitFilePath.string() << "to " << baseFilePath.string();
+            }
+        }
+    }
+
+    return;
+}
+
 void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const unsigned int& tid)
 {
     std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
@@ -4345,6 +4589,25 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
         }
         else
         {
+            if (zfe.isSplitFile)
+            {
+                if (boost::filesystem::exists(zfe.splitFileBasePath))
+                {
+                    msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file exists. Checking if it is same version.", MSGTYPE_INFO, msg_prefix));
+
+                    std::string crc32 = Util::getFileHashRange(zfe.splitFileBasePath, RHASH_CRC32, zfe.splitFileStartOffset, zfe.splitFileEndOffset);
+                    if (crc32 == Util::formattedString("%08x", zfe.crc32))
+                    {
+                        msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file is same version. Skipping file.", MSGTYPE_INFO, msg_prefix));
+                        continue;
+                    }
+                    else
+                    {
+                        msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file is different version. Continuing to download file.", MSGTYPE_INFO, msg_prefix));
+                    }
+                }
+            }
+
             if (boost::filesystem::exists(path))
             {
                 if (conf.bVerbose)

src/util.cpp

@@ -70,6 +70,70 @@ std::string Util::getFileHash(const std::string& filename, unsigned hash_id)
     return result;
 }
 
+std::string Util::getFileHashRange(const std::string& filepath, unsigned hash_id, off_t range_start, off_t range_end)
+{
+    char result[rhash_get_hash_length(hash_id) + 1];
+
+    if (!boost::filesystem::exists(filepath))
+        return result;
+
+    off_t filesize = boost::filesystem::file_size(filepath);
+
+    if (range_end == 0 || range_end > filesize)
+        range_end = filesize;
+
+    if (range_end < range_start)
+    {
+        off_t tmp = range_start;
+        range_start = range_end;
+        range_end = tmp;
+    }
+
+    off_t chunk_size = 10 << 20; // 10MB
+    off_t rangesize = range_end - range_start;
+    off_t remaining = rangesize % chunk_size;
+    int chunks = (remaining == 0) ? rangesize/chunk_size : (rangesize/chunk_size)+1;
+
+    rhash rhash_context;
+    rhash_context = rhash_init(hash_id);
+
+    FILE *infile = fopen(filepath.c_str(), "r");
+
+    for (int i = 0; i < chunks; i++)
+    {
+        off_t chunk_begin = range_start + i*chunk_size;
+        fseek(infile, chunk_begin, SEEK_SET);
+        if ((i == chunks-1) && (remaining != 0))
+            chunk_size = remaining;
+
+        unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *));
+        if (chunk == NULL)
+        {
+            std::cerr << "Memory error" << std::endl;
+            fclose(infile);
+            return result;
+        }
+        off_t size = fread(chunk, 1, chunk_size, infile);
+        if (size != chunk_size)
+        {
+            std::cerr << "Read error" << std::endl;
+            free(chunk);
+            fclose(infile);
+            return result;
+        }
+
+        rhash_update(rhash_context, chunk, chunk_size);
+        free(chunk);
+    }
+    fclose(infile);
+
+    rhash_final(rhash_context, NULL);
+    rhash_print(result, rhash_context, hash_id, RHPR_HEX);
+    rhash_free(rhash_context);
+
+    return result;
+}
+
 std::string Util::getChunkHash(unsigned char *chunk, uintmax_t chunk_size, unsigned hash_id)
 {
     unsigned char digest[rhash_get_digest_size(hash_id)];