Merge pull request #1885 from degasus/custom_texture

CustomTexture: new name format

Source/Core/Common/Hash.h

@@ -13,7 +13,7 @@ u32 HashAdler32(const u8* data, size_t len);         // Fairly accurate, slightl
 u32 HashFNV(const u8* ptr, int length);              // Another fast and decent hash
 u32 HashEctor(const u8* ptr, int length);            // JUNK. DO NOT USE FOR NEW THINGS
 u64 GetCRC32(const u8 *src, int len, u32 samples);   // SSE4.2 version of CRC32
-u64 GetHashHiresTexture(const u8 *src, int len, u32 samples);
+u64 GetHashHiresTexture(const u8 *src, int len, u32 samples = 0);
 u64 GetMurmurHash3(const u8 *src, int len, u32 samples);
 u64 GetHash64(const u8 *src, int len, u32 samples);
 void SetHash64Function();

Source/Core/VideoCommon/HiresTextures.cpp

@@ -3,9 +3,11 @@
 // Refer to the license.txt file included.
 
 #include <algorithm>
+#include <cinttypes>
 #include <cstring>
 #include <string>
 #include <utility>
+#include <xxhash.h>
 #include <SOIL/SOIL.h>
 
 #include "Common/CommonPaths.h"
@@ -16,13 +18,20 @@
 #include "Core/ConfigManager.h"
 
 #include "VideoCommon/HiresTextures.h"
+#include "VideoCommon/OnScreenDisplay.h"
 #include "VideoCommon/VideoConfig.h"
 
-std::unordered_map<std::string, std::string> HiresTexture::textureMap;
+static std::unordered_map<std::string, std::string> s_textureMap;
+static bool s_check_native_format;
+static bool s_check_new_format;
+
+static const std::string s_format_prefix = "tex1_";
 
 void HiresTexture::Init(const std::string& gameCode)
 {
-	textureMap.clear();
+	s_textureMap.clear();
+	s_check_native_format = false;
+	s_check_new_format = false;
 
 	CFileSearch::XStringVector Directories;
 
@@ -66,36 +75,172 @@ void HiresTexture::Init(const std::string& gameCode)
 	const CFileSearch::XStringVector& rFilenames = FileSearch.GetFileNames();
 
 	const std::string code = StringFromFormat("%s_", gameCode.c_str());
+	const std::string code2 = "";
 
-	if (rFilenames.size() > 0)
-	{
 	for (auto& rFilename : rFilenames)
 	{
 		std::string FileName;
 		SplitPath(rFilename, nullptr, &FileName, nullptr);
 
-			if (FileName.substr(0, code.length()).compare(code) == 0 && textureMap.find(FileName) == textureMap.end())
+		if (FileName.substr(0, code.length()) == code)
-				textureMap.insert(std::map<std::string, std::string>::value_type(FileName, rFilename));
+		{
+			s_textureMap[FileName] = rFilename;
+			s_check_native_format = true;
+		}
+
+		if (FileName.substr(0, s_format_prefix.length()) == s_format_prefix)
+		{
+			s_textureMap[FileName] = rFilename;
+			s_check_new_format = true;
 		}
 	}
 }
 
-std::string HiresTexture::GenBaseName(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format)
+std::string HiresTexture::GenBaseName(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps, bool dump)
 {
+	std::string name = "";
+	bool convert = false;
+	if (!dump && s_check_native_format)
+	{
+		// try to load the old format first
 		u64 tex_hash = GetHashHiresTexture(texture, (int)texture_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
-	u64 tlut_hash = 0;
+		u64 tlut_hash = tlut_size ? GetHashHiresTexture(tlut, (int)tlut_size, g_ActiveConfig.iSafeTextureCache_ColorSamples) : 0;
-	u64 hash = tex_hash;
+		name = StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(), (u32)(tex_hash ^ tlut_hash), (u16)format);
-	if (tlut_size)
+		if (s_textureMap.find(name) != s_textureMap.end())
 		{
-		tlut_hash = GetHashHiresTexture(tlut, (int)tlut_size, g_ActiveConfig.iSafeTextureCache_ColorSamples);
+			if (g_ActiveConfig.bConvertHiresTextures)
-		hash ^= tlut_hash;
+				convert = true;
+			else
+				return name;
 		}
-	return StringFromFormat("%s_%08x_%i", SConfig::GetInstance().m_LocalCoreStartupParameter.m_strUniqueID.c_str(), (u32)hash, (u16)format);
 	}
 
-HiresTexture* HiresTexture::Search(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format)
+	if (dump || s_check_new_format || convert)
 	{
-	std::string base_filename = GenBaseName(texture, texture_size, tlut, tlut_size, width, height, format);
+		// checking for min/max on paletted textures
+		u32 min = 0xffff;
+		u32 max = 0;
+		switch(tlut_size)
+		{
+			case 0: break;
+			case 16 * 2:
+				for (size_t i = 0; i < texture_size; i++)
+				{
+					min = std::min<u32>(min, texture[i] & 0xf);
+					min = std::min<u32>(min, texture[i] >> 4);
+					max = std::max<u32>(max, texture[i] & 0xf);
+					max = std::max<u32>(max, texture[i] >> 4);
+				}
+				break;
+			case 256 * 2:
+				for (size_t i = 0; i < texture_size; i++)
+				{
+					min = std::min<u32>(min, texture[i]);
+					max = std::max<u32>(max, texture[i]);
+				}
+				break;
+			case 16384 * 2:
+				for (size_t i = 0; i < texture_size/2; i++)
+				{
+					min = std::min<u32>(min, Common::swap16(((u16*)texture)[i]) & 0x3fff);
+					max = std::max<u32>(max, Common::swap16(((u16*)texture)[i]) & 0x3fff);
+				}
+				break;
+		}
+		if (tlut_size > 0)
+		{
+			tlut_size = 2 * (max + 1 - min);
+			tlut += 2 * min;
+		}
+
+		u64 tex_hash = XXH64(texture, texture_size, 0);
+		u64 tlut_hash = tlut_size ? XXH64(tlut, tlut_size, 0) : 0;
+
+		std::string basename = s_format_prefix + StringFromFormat("%dx%d%s_%016" PRIx64, width, height, has_mipmaps ? "_m" : "", tex_hash);
+		std::string tlutname = tlut_size ? StringFromFormat("_%016" PRIx64, tlut_hash) : "";
+		std::string formatname = StringFromFormat("_%d", format);
+		std::string fullname = basename + tlutname + formatname;
+
+		for (int level = 0; level < 10 && convert; level++)
+		{
+			std::string oldname = name;
+			if (level)
+				oldname += StringFromFormat("_mip%d", level);
+
+			// skip not existing levels
+			if (s_textureMap.find(oldname) == s_textureMap.end())
+				continue;
+
+			for (int i = 0;; i++)
+			{
+				// for hash collisions, padd with an integer
+				std::string newname = fullname;
+				if (level)
+					newname += StringFromFormat("_mip%d", level);
+				if (i)
+					newname += StringFromFormat(".%d", i);
+
+				// new texture
+				if (s_textureMap.find(newname) == s_textureMap.end())
+				{
+					std::string src = s_textureMap[oldname];
+					size_t postfix = src.find_last_of('.');
+					std::string dst = src.substr(0, postfix - oldname.length()) + newname + src.substr(postfix, src.length() - postfix);
+					if (File::Rename(src, dst))
+					{
+						s_textureMap.erase(oldname);
+						s_textureMap[newname] = dst;
+						s_check_new_format = true;
+						OSD::AddMessage(StringFromFormat("Rename custom texture %s to %s", oldname.c_str(), newname.c_str()), 5000);
+					}
+					else
+					{
+						ERROR_LOG(VIDEO, "rename failed");
+					}
+					break;
+				}
+				else
+				{
+					// dst fail already exist, compare content
+					std::string a, b;
+					File::ReadFileToString(s_textureMap[oldname], a);
+					File::ReadFileToString(s_textureMap[newname], b);
+
+					if (a == b && a != "")
+					{
+						// equal, so remove
+						if (File::Delete(s_textureMap[oldname]))
+						{
+							s_textureMap.erase(oldname);
+							OSD::AddMessage(StringFromFormat("Delete double old custom texture %s", oldname.c_str()), 5000);
+						}
+						else
+						{
+							ERROR_LOG(VIDEO, "delete failed");
+						}
+						break;
+					}
+
+					// else continue in this loop with the next higher padding variable
+				}
+			}
+		}
+
+		// try to match a wildcard template
+		if (!dump && s_textureMap.find(basename + "_*" + formatname) != s_textureMap.end())
+			return basename + "_*" + formatname;
+
+		// else generate the complete texture
+		if (dump || s_textureMap.find(fullname) != s_textureMap.end())
+			return fullname;
+	}
+
+	return name;
+}
+
+HiresTexture* HiresTexture::Search(const u8* texture, size_t texture_size, const u8* tlut, size_t tlut_size, u32 width, u32 height, int format, bool has_mipmaps)
+{
+	std::string base_filename = GenBaseName(texture, texture_size, tlut, tlut_size, width, height, format, has_mipmaps);
 
 	HiresTexture* ret = nullptr;
 	for (int level = 0;; level++)
@@ -106,12 +251,12 @@ HiresTexture* HiresTexture::Search(const u8* texture, size_t texture_size, const
 			filename += StringFromFormat("_mip%u", level);
 		}
 
-		if (textureMap.find(filename) != textureMap.end())
+		if (s_textureMap.find(filename) != s_textureMap.end())
 		{
 			Level l;
 
 			File::IOFile file;
-			file.Open(textureMap[filename], "rb");
+			file.Open(s_textureMap[filename], "rb");
 			std::vector<u8> buffer(file.GetSize());
 			file.ReadBytes(buffer.data(), file.GetSize());
 

Source/Core/VideoCommon/HiresTextures.h

@@ -18,14 +18,15 @@ public:
 		const u8* texture, size_t texture_size,
 		const u8* tlut, size_t tlut_size,
 		u32 width, u32 height,
-		int format
+		int format, bool has_mipmaps
 	);
 
 	static std::string GenBaseName(
 		const u8* texture, size_t texture_size,
 		const u8* tlut, size_t tlut_size,
 		u32 width, u32 height,
-		int format
+		int format, bool has_mipmaps,
+		bool dump = false
 	);
 
 	~HiresTexture();
@@ -38,8 +39,6 @@ public:
 	};
 	std::vector<Level> m_levels;
 
-	static std::unordered_map<std::string, std::string> textureMap;
-
 private:
 	HiresTexture() {}
 

Source/Core/VideoCommon/TextureCacheBase.cpp

@@ -304,8 +304,8 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 	const int texformat = tex.texImage0[id].format;
 	const u32 tlutaddr = tex.texTlut[id].tmem_offset << 9;
 	const u32 tlutfmt = tex.texTlut[id].tlut_format;
-	const bool use_mipmaps = (tex.texMode0[id].min_filter & 3) != 0;
 	u32 tex_levels = (tex.texMode1[id].max_lod + 0xf) / 0x10 + 1;
+	const bool use_mipmaps = (tex.texMode0[id].min_filter & 3) != 0 && tex_levels > 0;
 	const bool from_tmem = tex.texImage1[id].image_type != 0;
 
 	if (0 == address)
@@ -410,7 +410,7 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 			src_data, texture_size,
 			&texMem[tlutaddr], palette_size,
 			width, height,
-			texformat
+			texformat, use_mipmaps
 		));
 
 		if (hires_tex)
@@ -476,7 +476,8 @@ TextureCache::TCacheEntryBase* TextureCache::Load(const u32 stage)
 			src_data, texture_size,
 			&texMem[tlutaddr], palette_size,
 			width, height,
-			texformat
+			texformat, use_mipmaps,
+			true
 		);
 		DumpTexture(entry, basename, 0);
 	}

Source/Core/VideoCommon/VideoConfig.cpp

@@ -70,6 +70,7 @@ void VideoConfig::Load(const std::string& ini_file)
 	settings->Get("ShowEFBCopyRegions", &bShowEFBCopyRegions, false);
 	settings->Get("DumpTextures", &bDumpTextures, 0);
 	settings->Get("HiresTextures", &bHiresTextures, 0);
+	settings->Get("ConvertHiresTextures", &bConvertHiresTextures, 0);
 	settings->Get("DumpEFBTarget", &bDumpEFBTarget, 0);
 	settings->Get("FreeLook", &bFreeLook, 0);
 	settings->Get("UseFFV1", &bUseFFV1, 0);
@@ -150,6 +151,7 @@ void VideoConfig::GameIniLoad()
 	CHECK_SETTING("Video_Settings", "UseRealXFB", bUseRealXFB);
 	CHECK_SETTING("Video_Settings", "SafeTextureCacheColorSamples", iSafeTextureCache_ColorSamples);
 	CHECK_SETTING("Video_Settings", "HiresTextures", bHiresTextures);
+	CHECK_SETTING("Video_Settings", "ConvertHiresTextures", bConvertHiresTextures);
 	CHECK_SETTING("Video_Settings", "EnablePixelLighting", bEnablePixelLighting);
 	CHECK_SETTING("Video_Settings", "FastDepthCalc", bFastDepthCalc);
 	CHECK_SETTING("Video_Settings", "MSAA", iMultisampleMode);
@@ -257,6 +259,7 @@ void VideoConfig::Save(const std::string& ini_file)
 	settings->Set("OverlayProjStats", bOverlayProjStats);
 	settings->Set("DumpTextures", bDumpTextures);
 	settings->Set("HiresTextures", bHiresTextures);
+	settings->Set("ConvertHiresTextures", bConvertHiresTextures);
 	settings->Set("DumpEFBTarget", bDumpEFBTarget);
 	settings->Set("FreeLook", bFreeLook);
 	settings->Set("UseFFV1", bUseFFV1);

Source/Core/VideoCommon/VideoConfig.h

@@ -103,6 +103,7 @@ struct VideoConfig final
 	// Utility
 	bool bDumpTextures;
 	bool bHiresTextures;
+	bool bConvertHiresTextures;
 	bool bDumpEFBTarget;
 	bool bUseFFV1;
 	bool bFreeLook;