Mirrors/lgogdownloader
2
0
Fork 0
mirror of https://github.com/Sude-/lgogdownloader.git synced 2024-11-20 03:39:17 +01:00
Code Issues Packages Projects Releases Wiki Activity

Galaxy: Add split file support to MojoSetup hack

Browse Source
This commit is contained in:
Sude 2020-02-19 13:49:06 +02:00
parent 0a7648d80b
commit 8caca0e62a
4 changed files with 283 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
include/downloader.h
View File

@ -68,19 +68,28 @@ struct ChunkMemoryStruct
typedef struct
{
std::string filepath;
off_t comp_size;
off_t uncomp_size;
off_t start_offset_zip;
off_t start_offset_mojosetup;
off_t end_offset;
uint16_t file_attributes;
uint32_t crc32;
time_t timestamp;
std::string filepath;
off_t comp_size;
off_t uncomp_size;
off_t start_offset_zip;
off_t start_offset_mojosetup;
off_t end_offset;
uint16_t file_attributes;
uint32_t crc32;
time_t timestamp;
std::string installer_url;
std::string installer_url;
// For split file handling
bool isSplitFile = false;
std::string splitFileBasePath;
std::string splitFilePartExt;
off_t splitFileStartOffset;
off_t splitFileEndOffset;
} zipFileEntry;
typedef std::map<std::string,std::vector<zipFileEntry>> splitFilesMap;
class Downloader
{
public:
@ -140,6 +149,7 @@ class Downloader
std::vector<std::string> galaxyGetOrphanedFiles(const std::vector<galaxyDepotItem>& items, const std::string& install_path);
static void processGalaxyDownloadQueue(const std::string& install_path, Config conf, const unsigned int& tid);
void galaxyInstallGame_MojoSetupHack(const std::string& product_id);
void galaxyInstallGame_MojoSetupHack_CombineSplitFiles(const splitFilesMap& mSplitFiles, const bool& bAppendtoFirst = false);
static void processGalaxyDownloadQueue_MojoSetupHack(Config conf, const unsigned int& tid);
int mojoSetupGetFileVector(const gameFile& gf, std::vector<zipFileEntry>& vFiles);
std::string getGalaxyInstallDirectory(galaxyAPI *galaxyHandle, const Json::Value& manifest);

1
include/util.h
View File

@ -54,6 +54,7 @@ namespace Util
std::string makeFilepath(const std::string& directory, const std::string& path, const std::string& gamename, std::string subdirectory = "", const unsigned int& platformId = 0, const std::string& dlcname = "");
std::string makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory = "");
std::string getFileHash(const std::string& filename, unsigned hash_id);
std::string getFileHashRange(const std::string& filepath, unsigned hash_id, off_t range_start = 0, off_t range_end = 0);
std::string getChunkHash(unsigned char* chunk, uintmax_t chunk_size, unsigned hash_id);
int createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir = std::string());
int getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf, std::string directory = std::string());

199
src/downloader.cpp
View File

@ -4192,6 +4192,7 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
std::vector<zipFileEntry> vZipDirectories;
std::vector<zipFileEntry> vZipFiles;
std::vector<zipFileEntry> vZipFilesSymlink;
std::vector<zipFileEntry> vZipFilesSplit;
for (std::uintmax_t i = 0; i < zipFileEntries.size(); ++i)
{
// Ignore all files and directories that are not in "data/noarch/" directory
@ -4208,7 +4209,22 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
else if (ZipUtil::isSymlink(zfe.file_attributes))
vZipFilesSymlink.push_back(zfe);
else
vZipFiles.push_back(zfe);
{
// Check for split files
boost::regex expression("^(.*)(\\.split\\d+)$");
boost::match_results<std::string::const_iterator> what;
if (boost::regex_search(zfe.filepath, what, expression))
{
zfe.isSplitFile = true;
zfe.splitFileBasePath = what[1];
zfe.splitFilePartExt = what[2];
vZipFilesSplit.push_back(zfe);
}
else
{
vZipFiles.push_back(zfe);
}
}
}
// Create directories
@ -4224,6 +4240,42 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
}
}
// Set start and end offsets for split files
// Create map of split files for combining them later
splitFilesMap mSplitFiles;
if (!vZipFilesSplit.empty())
{
std::sort(vZipFilesSplit.begin(), vZipFilesSplit.end(), [](const zipFileEntry& i, const zipFileEntry& j) -> bool { return i.filepath < j.filepath; });
std::string prevBasePath = "";
off_t prevEndOffset = 0;
for (auto& zfe : vZipFilesSplit)
{
if (zfe.splitFileBasePath == prevBasePath)
zfe.splitFileStartOffset = prevEndOffset;
else
zfe.splitFileStartOffset = 0;
zfe.splitFileEndOffset = zfe.splitFileStartOffset + zfe.uncomp_size;
prevBasePath = zfe.splitFileBasePath;
prevEndOffset = zfe.splitFileEndOffset;
if (mSplitFiles.count(zfe.splitFileBasePath) > 0)
{
mSplitFiles[zfe.splitFileBasePath].push_back(zfe);
}
else
{
std::vector<zipFileEntry> vec;
vec.push_back(zfe);
mSplitFiles[zfe.splitFileBasePath] = vec;
}
}
vZipFiles.insert(std::end(vZipFiles), std::begin(vZipFilesSplit), std::end(vZipFilesSplit));
}
// Add files to download queue
for (std::uintmax_t i = 0; i < vZipFiles.size(); ++i)
{
@ -4259,6 +4311,12 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
vThreads.clear();
vDownloadInfo.clear();
// Combine split files
if (!mSplitFiles.empty())
{
this->galaxyInstallGame_MojoSetupHack_CombineSplitFiles(mSplitFiles, true);
}
}
else
{
@ -4266,6 +4324,126 @@ void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id)
}
}
void Downloader::galaxyInstallGame_MojoSetupHack_CombineSplitFiles(const splitFilesMap& mSplitFiles, const bool& bAppendToFirst)
{
for (const auto& baseFile : mSplitFiles)
{
// Check that all parts exist
bool bAllPartsExist = true;
for (const auto& splitFile : baseFile.second)
{
if (!boost::filesystem::exists(splitFile.filepath))
{
bAllPartsExist = false;
break;
}
}
bool bBaseFileExists = boost::filesystem::exists(baseFile.first);
if (!bAllPartsExist)
{
if (bBaseFileExists)
{
// Base file exist and we're missing parts.
// This should mean that we already have complete file.
// So we can safely skip this file without informing the user
continue;
}
else
{
// Base file doesn't exist and we're missing parts. Print message about it before skipping file.
std::cout << baseFile.first << " is missing parts. Skipping this file." << std::endl;
continue;
}
}
// Delete base file if it already exists
if (bBaseFileExists)
{
std::cout << baseFile.first << " already exists. Deleting old file." << std::endl;
if (!boost::filesystem::remove(baseFile.first))
{
std::cout << baseFile.first << ": Failed to delete" << std::endl;
continue;
}
}
std::cout << "Beginning to combine " << baseFile.first << std::endl;
std::ofstream ofs;
// Create base file for appending if we aren't appending to first part
if (!bAppendToFirst)
{
ofs.open(baseFile.first, std::ios_base::binary | std::ios_base::app);
if (!ofs.is_open())
{
std::cout << "Failed to create " << baseFile.first << std::endl;
continue;
}
}
for (const auto& splitFile : baseFile.second)
{
std::cout << "\t" << splitFile.filepath << std::endl;
// Append to first file is set and current file is first in vector.
// Open file for appending and continue to next file
if (bAppendToFirst && (&splitFile == &baseFile.second.front()))
{
ofs.open(splitFile.filepath, std::ios_base::binary | std::ios_base::app);
if (!ofs.is_open())
{
std::cout << "Failed to open " << splitFile.filepath << std::endl;
break;
}
continue;
}
std::ifstream ifs(splitFile.filepath, std::ios_base::binary);
if (!ifs)
{
std::cout << "Failed to open " << splitFile.filepath << ". Deleting incomplete file." << std::endl;
ofs.close();
if (!boost::filesystem::remove(baseFile.first))
{
std::cout << baseFile.first << ": Failed to delete" << std::endl;
}
break;
}
ofs << ifs.rdbuf();
ifs.close();
// Delete split file
if (!boost::filesystem::remove(splitFile.filepath))
{
std::cout << splitFile.filepath << ": Failed to delete" << std::endl;
}
}
if (ofs)
ofs.close();
// Appending to first file so we must rename it
if (bAppendToFirst)
{
boost::filesystem::path splitFilePath = baseFile.second.front().filepath;
boost::filesystem::path baseFilePath = baseFile.first;
boost::system::error_code ec;
boost::filesystem::rename(splitFilePath, baseFilePath, ec);
if (ec)
{
std::cout << "Failed to rename " << splitFilePath.string() << "to " << baseFilePath.string();
}
}
}
return;
}
void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const unsigned int& tid)
{
std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]";
@ -4345,6 +4523,25 @@ void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const uns
}
else
{
if (zfe.isSplitFile)
{
if (boost::filesystem::exists(zfe.splitFileBasePath))
{
msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file exists. Checking if it is same version.", MSGTYPE_INFO, msg_prefix));
std::string crc32 = Util::getFileHashRange(zfe.splitFileBasePath, RHASH_CRC32, zfe.splitFileStartOffset, zfe.splitFileEndOffset);
if (crc32 == Util::formattedString("%08x", zfe.crc32))
{
msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file is same version. Skipping file.", MSGTYPE_INFO, msg_prefix));
continue;
}
else
{
msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file is different version. Continuing to download file.", MSGTYPE_INFO, msg_prefix));
}
}
}
if (boost::filesystem::exists(path))
{
if (conf.bVerbose)

64
src/util.cpp
View File

@ -70,6 +70,70 @@ std::string Util::getFileHash(const std::string& filename, unsigned hash_id)
return result;
}
std::string Util::getFileHashRange(const std::string& filepath, unsigned hash_id, off_t range_start, off_t range_end)
{
char result[rhash_get_hash_length(hash_id) + 1];
if (!boost::filesystem::exists(filepath))
return result;
off_t filesize = boost::filesystem::file_size(filepath);
if (range_end == 0 || range_end > filesize)
range_end = filesize;
if (range_end < range_start)
{
off_t tmp = range_start;
range_start = range_end;
range_end = tmp;
}
off_t chunk_size = 10 << 20; // 10MB
off_t rangesize = range_end - range_start;
off_t remaining = rangesize % chunk_size;
int chunks = (remaining == 0) ? rangesize/chunk_size : (rangesize/chunk_size)+1;
rhash rhash_context;
rhash_context = rhash_init(hash_id);
FILE *infile = fopen(filepath.c_str(), "r");
for (int i = 0; i < chunks; i++)
{
off_t chunk_begin = range_start + i*chunk_size;
fseek(infile, chunk_begin, SEEK_SET);
if ((i == chunks-1) && (remaining != 0))
chunk_size = remaining;
unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *));
if (chunk == NULL)
{
std::cerr << "Memory error" << std::endl;
fclose(infile);
return result;
}
off_t size = fread(chunk, 1, chunk_size, infile);
if (size != chunk_size)
{
std::cerr << "Read error" << std::endl;
free(chunk);
fclose(infile);
return result;
}
rhash_update(rhash_context, chunk, chunk_size);
free(chunk);
}
fclose(infile);
rhash_final(rhash_context, NULL);
rhash_print(result, rhash_context, hash_id, RHPR_HEX);
rhash_free(rhash_context);
return result;
}
std::string Util::getChunkHash(unsigned char *chunk, uintmax_t chunk_size, unsigned hash_id)
{
unsigned char digest[rhash_get_digest_size(hash_id)];