mirror of
https://github.com/Sude-/lgogdownloader.git
synced 2025-02-01 13:32:32 +01:00
Galaxy: Add small files container support
Makes the downloader use small files container when installing games via Galaxy API. Instead of downloading a lot of very small files we download a file that contains all the small files and extract it when everything is downloaded.
This commit is contained in:
Sude
parent
1b14b1129c
commit
70a4437c3c
@ -38,6 +38,10 @@ struct galaxyDepotItem
|
||||
std::string md5;
|
||||
std::string product_id;
|
||||
bool isDependency = false;
|
||||
bool isSmallFilesContainer = false;
|
||||
bool isInSFC = false;
|
||||
uintmax_t sfc_offset;
|
||||
uintmax_t sfc_size;
|
||||
};
|
||||
|
||||
class galaxyAPI
|
||||
|
||||
@ -4013,13 +4013,17 @@ std::vector<galaxyDepotItem> Downloader::galaxyGetDepotItemVectorFromJson(const
|
||||
}
|
||||
}
|
||||
|
||||
// Set product id for items
|
||||
// Set product id for items and add product id to small files container name
|
||||
for (auto it = items.begin(); it != items.end(); ++it)
|
||||
{
|
||||
if (it->product_id.empty())
|
||||
{
|
||||
it->product_id = product_id;
|
||||
}
|
||||
if (it->isSmallFilesContainer)
|
||||
{
|
||||
it->path += "_" + it->product_id;
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
@ -4105,6 +4109,49 @@ void Downloader::galaxyInstallGameById(const std::string& product_id, const std:
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<galaxyDepotItem> items_smallfiles;
|
||||
std::vector<galaxyDepotItem> sfc_vector;
|
||||
bool bUseSmallFilesContainer = true;
|
||||
for (auto item : items)
|
||||
{
|
||||
if (item.isInSFC)
|
||||
{
|
||||
std::string item_install_path = install_path + "/" + item.path;
|
||||
if (boost::filesystem::exists(item_install_path))
|
||||
{
|
||||
bUseSmallFilesContainer = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!bUseSmallFilesContainer)
|
||||
{
|
||||
for (std::vector<galaxyDepotItem>::iterator it = items.begin(); it != items.end();)
|
||||
{
|
||||
if (it->isSmallFilesContainer)
|
||||
it = items.erase(it);
|
||||
else
|
||||
++it;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (std::vector<galaxyDepotItem>::iterator it = items.begin(); it != items.end();)
|
||||
{
|
||||
if (it->isSmallFilesContainer)
|
||||
sfc_vector.push_back(*it);
|
||||
|
||||
if (it->isInSFC)
|
||||
{
|
||||
items_smallfiles.push_back(*it);
|
||||
it = items.erase(it);
|
||||
}
|
||||
else
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for differences between previously installed build and new build
|
||||
std::vector<galaxyDepotItem> items_old;
|
||||
|
||||
@ -4227,7 +4274,66 @@ void Downloader::galaxyInstallGameById(const std::string& product_id, const std:
|
||||
vThreads.clear();
|
||||
vDownloadInfo.clear();
|
||||
|
||||
if (bUseSmallFilesContainer)
|
||||
{
|
||||
for (auto container : sfc_vector)
|
||||
{
|
||||
std::string container_install_path = install_path + "/" + container.path;
|
||||
if (!boost::filesystem::exists(container_install_path))
|
||||
continue;
|
||||
|
||||
std::cout << "Extracting small files container " << container_install_path << std::endl;
|
||||
|
||||
for (auto item : items_smallfiles)
|
||||
{
|
||||
if (item.product_id != container.product_id)
|
||||
continue;
|
||||
|
||||
std::string item_install_path = install_path + "/" + item.path;
|
||||
std::cout << item_install_path << std::endl;
|
||||
std::ifstream sfc(container_install_path, std::ifstream::binary);
|
||||
if (sfc)
|
||||
{
|
||||
sfc.seekg(item.sfc_offset, sfc.beg);
|
||||
char *filecontents = (char *) malloc(item.sfc_size);
|
||||
sfc.read(filecontents, item.sfc_size);
|
||||
sfc.close();
|
||||
|
||||
// Check that directory exists and create it
|
||||
boost::filesystem::path path = item_install_path;
|
||||
boost::filesystem::path directory = path.parent_path();
|
||||
if (!boost::filesystem::exists(directory))
|
||||
{
|
||||
if (!boost::filesystem::create_directories(directory))
|
||||
{
|
||||
std::cout << "Failed to create directory: " << directory << std::endl;
|
||||
free(filecontents);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
std::ofstream output(item_install_path, std::ofstream::binary);
|
||||
if (output)
|
||||
{
|
||||
output.write(filecontents, item.sfc_size);
|
||||
output.close();
|
||||
}
|
||||
free(filecontents);
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Deleting small files container " << container_install_path << std::endl;
|
||||
if (!boost::filesystem::remove(container_install_path))
|
||||
std::cerr << "Failed to delete " << container_install_path << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Checking for orphaned files" << std::endl;
|
||||
if (bUseSmallFilesContainer)
|
||||
{
|
||||
// Add small files back to items vector for ophan checking
|
||||
items.insert(std::end(items), std::begin(items_smallfiles), std::end(items_smallfiles));
|
||||
}
|
||||
std::vector<std::string> orphans = this->galaxyGetOrphanedFiles(items, install_path);
|
||||
std::cout << "\t" << orphans.size() << " orphaned files" << std::endl;
|
||||
for (unsigned int i = 0; i < orphans.size(); ++i)
|
||||
|
||||
@ -255,6 +255,45 @@ std::vector<galaxyDepotItem> galaxyAPI::getDepotItemsVector(const std::string& h
|
||||
Json::Value json = this->getManifestV2(hash, is_dependency);
|
||||
|
||||
std::vector<galaxyDepotItem> items;
|
||||
if (json["depot"].isMember("smallFilesContainer"))
|
||||
{
|
||||
if (json["depot"]["smallFilesContainer"]["chunks"].isArray())
|
||||
{
|
||||
galaxyDepotItem item;
|
||||
item.totalSizeCompressed = 0;
|
||||
item.totalSizeUncompressed = 0;
|
||||
item.path = "galaxy_smallfilescontainer";
|
||||
item.isDependency = is_dependency;
|
||||
item.isSmallFilesContainer = true;
|
||||
|
||||
for (unsigned int i = 0; i < json["depot"]["smallFilesContainer"]["chunks"].size(); ++i)
|
||||
{
|
||||
Json::Value json_chunk = json["depot"]["smallFilesContainer"]["chunks"][i];
|
||||
|
||||
galaxyDepotItemChunk chunk;
|
||||
chunk.md5_compressed = json_chunk["compressedMd5"].asString();
|
||||
chunk.md5_uncompressed = json_chunk["md5"].asString();
|
||||
chunk.size_compressed = json_chunk["compressedSize"].asLargestUInt();
|
||||
chunk.size_uncompressed = json_chunk["size"].asLargestUInt();
|
||||
|
||||
chunk.offset_compressed = item.totalSizeCompressed;
|
||||
chunk.offset_uncompressed = item.totalSizeUncompressed;
|
||||
|
||||
item.totalSizeCompressed += chunk.size_compressed;
|
||||
item.totalSizeUncompressed += chunk.size_uncompressed;
|
||||
item.chunks.push_back(chunk);
|
||||
}
|
||||
|
||||
if (json["depot"]["smallFilesContainer"].isMember("md5"))
|
||||
item.md5 = json["depot"]["smallFilesContainer"]["md5"].asString();
|
||||
else if (json["depot"]["smallFilesContainer"]["chunks"].size() == 1)
|
||||
item.md5 = json["depot"]["smallFilesContainer"]["chunks"][0]["md5"].asString();
|
||||
else
|
||||
item.md5 = std::string();
|
||||
|
||||
items.push_back(item);
|
||||
}
|
||||
}
|
||||
|
||||
for (unsigned int i = 0; i < json["depot"]["items"].size(); ++i)
|
||||
{
|
||||
@ -266,14 +305,23 @@ std::vector<galaxyDepotItem> galaxyAPI::getDepotItemsVector(const std::string& h
|
||||
item.path = json["depot"]["items"][i]["path"].asString();
|
||||
item.isDependency = is_dependency;
|
||||
|
||||
if (json["depot"]["items"][i].isMember("sfcRef"))
|
||||
{
|
||||
item.isInSFC = true;
|
||||
item.sfc_offset = json["depot"]["items"][i]["sfcRef"]["offset"].asLargestUInt();
|
||||
item.sfc_size = json["depot"]["items"][i]["sfcRef"]["size"].asLargestUInt();
|
||||
}
|
||||
|
||||
while (Util::replaceString(item.path, "\\", "/"));
|
||||
for (unsigned int j = 0; j < json["depot"]["items"][i]["chunks"].size(); ++j)
|
||||
{
|
||||
Json::Value json_chunk = json["depot"]["items"][i]["chunks"][j];
|
||||
|
||||
galaxyDepotItemChunk chunk;
|
||||
chunk.md5_compressed = json["depot"]["items"][i]["chunks"][j]["compressedMd5"].asString();
|
||||
chunk.md5_uncompressed = json["depot"]["items"][i]["chunks"][j]["md5"].asString();
|
||||
chunk.size_compressed = json["depot"]["items"][i]["chunks"][j]["compressedSize"].asLargestUInt();
|
||||
chunk.size_uncompressed = json["depot"]["items"][i]["chunks"][j]["size"].asLargestUInt();
|
||||
chunk.md5_compressed = json_chunk["compressedMd5"].asString();
|
||||
chunk.md5_uncompressed = json_chunk["md5"].asString();
|
||||
chunk.size_compressed = json_chunk["compressedSize"].asLargestUInt();
|
||||
chunk.size_uncompressed = json_chunk["size"].asLargestUInt();
|
||||
|
||||
chunk.offset_compressed = item.totalSizeCompressed;
|
||||
chunk.offset_uncompressed = item.totalSizeUncompressed;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user