From 86910f406e3680c4cb01f37e5f29f894a25342cb Mon Sep 17 00:00:00 2001 From: JosJuice Date: Wed, 16 Aug 2023 09:54:24 +0200 Subject: [PATCH] VideoCommon: Fix std::filesystem::path encoding conversion In std::string, you can store strings using any encoding, but in Dolphin we have decided to use UTF-8. The problem is that if you convert between std::string and std::filesystem::path using the built-in methods, the standard library will make up its own assumption of what encoding you're using in the std::string. On most OSes this is UTF-8, but on Windows it's whatever the user's code page is. What I believe is the C++ standard authors' intended solution to this is to use std::u8string instead of std::string, but that's a big hassle to move over to, because there's no convenient way to convert between std::string and std::u8string. Instead, in Dolphin, we have added helper functions that convert between std::string and std::filesystem::path in the manner we want. You *always* have to use these when converting between std::string and std::filesystem::path, otherwise we get these kinds of encoding problems that we've been having with custom textures. Fixes https://bugs.dolphin-emu.org/issues/13328. --- .../Assets/DirectFilesystemAssetLibrary.cpp | 30 +++++++++---------- Source/Core/VideoCommon/HiresTextures.cpp | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp index cc21549e54..3d92c7fb82 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp @@ -103,18 +103,18 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadPixelShader(const } const auto approx_mem_size = metadata_size + shader_size; - if (!File::ReadFileToString(shader->second.string(), data->m_shader_source)) + if (!File::ReadFileToString(PathToString(shader->second), data->m_shader_source)) { ERROR_LOG_FMT(VIDEO, "Asset '{}' error - failed to load the shader file '{}',", asset_id, - shader->second.string()); + PathToString(shader->second)); return {}; } std::string json_data; - if (!File::ReadFileToString(metadata->second.string(), json_data)) + if (!File::ReadFileToString(PathToString(metadata->second), json_data)) { ERROR_LOG_FMT(VIDEO, "Asset '{}' error - failed to load the json file '{}',", asset_id, - metadata->second.string()); + PathToString(metadata->second)); return {}; } @@ -125,7 +125,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadPixelShader(const { ERROR_LOG_FMT(VIDEO, "Asset '{}' error - failed to load the json file '{}', due to parse error: {}", - asset_id, metadata->second.string(), error); + asset_id, PathToString(metadata->second), error); return {}; } if (!root.is()) @@ -133,7 +133,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadPixelShader(const ERROR_LOG_FMT( VIDEO, "Asset '{}' error - failed to load the json file '{}', due to root not being an object!", - asset_id, metadata->second.string()); + asset_id, PathToString(metadata->second)); return {}; } @@ -159,10 +159,10 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMaterial(const As const auto& asset_path = asset_map.begin()->second; std::string json_data; - if (!File::ReadFileToString(asset_path.string(), json_data)) + if (!File::ReadFileToString(PathToString(asset_path), json_data)) { ERROR_LOG_FMT(VIDEO, "Asset '{}' error - material failed to load the json file '{}',", - asset_id, asset_path.string()); + asset_id, PathToString(asset_path)); return {}; } @@ -174,7 +174,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMaterial(const As ERROR_LOG_FMT( VIDEO, "Asset '{}' error - material failed to load the json file '{}', due to parse error: {}", - asset_id, asset_path.string(), error); + asset_id, PathToString(asset_path), error); return {}; } if (!root.is()) @@ -182,7 +182,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMaterial(const As ERROR_LOG_FMT(VIDEO, "Asset '{}' error - material failed to load the json file '{}', due to root not " "being an object!", - asset_id, asset_path.string()); + asset_id, PathToString(asset_path)); return {}; } @@ -193,7 +193,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadMaterial(const As ERROR_LOG_FMT(VIDEO, "Asset '{}' error - material failed to load the json file '{}', as material " "json could not be parsed!", - asset_id, asset_path.string()); + asset_id, PathToString(asset_path)); return {}; } @@ -222,11 +222,11 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass asset_id, ec); return {}; } - auto ext = asset_path.extension().string(); + auto ext = PathToString(asset_path.extension()); Common::ToLower(&ext); if (ext == ".dds") { - if (!LoadDDSTexture(data, asset_path.string())) + if (!LoadDDSTexture(data, PathToString(asset_path))) { ERROR_LOG_FMT(VIDEO, "Asset '{}' error - could not load dds texture!", asset_id); return {}; @@ -243,7 +243,7 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass if (data->m_levels.empty()) data->m_levels.push_back({}); - if (!LoadPNGTexture(&data->m_levels[0], asset_path.string())) + if (!LoadPNGTexture(&data->m_levels[0], PathToString(asset_path))) { ERROR_LOG_FMT(VIDEO, "Asset '{}' error - could not load png texture!", asset_id); return {}; @@ -275,7 +275,7 @@ bool DirectFilesystemAssetLibrary::LoadMips(const std::filesystem::path& asset_p std::string path; std::string filename; std::string extension; - SplitPath(asset_path.string(), &path, &filename, &extension); + SplitPath(PathToString(asset_path), &path, &filename, &extension); std::string extension_lower = extension; Common::ToLower(&extension_lower); diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp index f9f3134e35..6a489973a5 100644 --- a/Source/Core/VideoCommon/HiresTextures.cpp +++ b/Source/Core/VideoCommon/HiresTextures.cpp @@ -131,7 +131,7 @@ void HiresTexture::Update() // Since this is just a texture (single file) the mapper doesn't really matter // just provide a string s_file_library->SetAssetIDMapData( - filename, std::map{{"", path}}); + filename, std::map{{"", StringToPath(path)}}); if (g_ActiveConfig.bCacheHiresTextures) {