Latte+GL+VK: Improve handling of gfx pack texture overwrite format

Graphic packs can overwrite the format of a texture (e.g. for higher bitdepth to lessen banding) but the code for this wasn't correctly working anymore.

- Fixes overwrite format being ignored for texture views on Vulkan backend
- Fixes overwrite format not being used for texture views on OpenGL

Format aliasing is complicated enough as it is, even without overwrites, so this adds a new rule to make behavior more well defined: If two textures share memory but only one uses an overwrite format, then they are no longer synchronized and are considered separate textures.

Bonus fixes for OpenGL:
- Use fbo 0 instead of -1 as the default. This silences some warnings in debug output
- On OpenGL, bind new framebuffers on handle generation so they are considered created

src/Cafe/HW/Latte/Core/LatteTexture.cpp

@@ -434,6 +434,11 @@ void LatteTexture_SyncSlice(LatteTexture* srcTexture, sint32 srcSliceIndex, sint
 	sint32 dstWidth = dstTexture->width;
 	sint32 dstHeight = dstTexture->height;
 
+	if(srcTexture->overwriteInfo.hasFormatOverwrite != dstTexture->overwriteInfo.hasFormatOverwrite)
+		return; // dont sync: format overwrite state needs to match. Not strictly necessary but it simplifies logic down the road
+	else if(srcTexture->overwriteInfo.hasFormatOverwrite && srcTexture->overwriteInfo.format != dstTexture->overwriteInfo.format)
+		return; // both are overwritten but with different formats
+
 	if (srcMipIndex == 0 && dstMipIndex == 0 && (srcTexture->tileMode == Latte::E_HWTILEMODE::TM_LINEAR_ALIGNED || srcTexture->tileMode == Latte::E_HWTILEMODE::TM_1D_TILED_THIN1) && srcTexture->height > dstTexture->height && (srcTexture->height % dstTexture->height) == 0)
 	{
 		bool isMatch = srcTexture->tileMode == Latte::E_HWTILEMODE::TM_LINEAR_ALIGNED;
@@ -816,6 +821,12 @@ VIEWCOMPATIBILITY LatteTexture_CanTextureBeRepresentedAsView(LatteTexture* baseT
 {
 	relativeMipIndex = 0;
 	relativeSliceIndex = 0;
+	if (baseTexture->overwriteInfo.hasFormatOverwrite)
+	{
+		// if the base format is overwritten, then we only allow aliasing if the view format matches the base format
+		if (baseTexture->format != format)
+			return VIEW_NOT_COMPATIBLE;
+	}
 	if (LatteTexture_IsFormatViewCompatible(baseTexture->format, format) == false)
 		return VIEW_NOT_COMPATIBLE;
 	if (baseTexture->physAddress == physAddr && baseTexture->pitch == pitch)

src/Cafe/HW/Latte/Renderer/OpenGL/LatteTextureGL.cpp

@@ -26,7 +26,7 @@ LatteTextureGL::LatteTextureGL(Latte::E_DIM dim, MPTR physAddress, MPTR physMipA
 	GenerateEmptyTextureFromGX2Dim(dim, this->glId_texture, this->glTexTarget, true);
 	// set format info
 	FormatInfoGL glFormatInfo;
-	GetOpenGLFormatInfo(isDepth, format, dim, &glFormatInfo);
+	GetOpenGLFormatInfo(isDepth, overwriteInfo.hasFormatOverwrite ? (Latte::E_GX2SURFFMT)overwriteInfo.format : format, dim, &glFormatInfo);
 	this->glInternalFormat = glFormatInfo.glInternalFormat;
 	this->isAlternativeFormat = glFormatInfo.isUsingAlternativeFormat;
 	this->hasStencil = glFormatInfo.hasStencil; // todo - should get this from the GX2 format?

src/Cafe/HW/Latte/Renderer/OpenGL/LatteTextureViewGL.cpp

@@ -55,12 +55,16 @@ LatteTextureViewGL::~LatteTextureViewGL()
 void LatteTextureViewGL::InitAliasView()
 {
 	const auto texture = (LatteTextureGL*)baseTexture;
-	// get internal format
+	// compute internal format
-	if (baseTexture->isDepth)
+	if(texture->overwriteInfo.hasFormatOverwrite)
+	{
+		cemu_assert_debug(format == texture->format);
+		glInternalFormat = texture->glInternalFormat; // for format overwrite no aliasing is allowed and thus we always inherit the internal format of the base texture
+	}
+	else if (baseTexture->isDepth)
 	{
 		// depth is handled differently
-		cemuLog_logDebug(LogType::Force, "Creating depth view");
+		cemu_assert(format == texture->format); // is depth alias with different format intended?
-		cemu_assert(format == texture->format); // todo
 		glInternalFormat = texture->glInternalFormat;
 	}
 	else
@@ -73,7 +77,7 @@ void LatteTextureViewGL::InitAliasView()
 	catchOpenGLError();
 	if (firstMip >= texture->maxPossibleMipLevels)
 	{
-		cemuLog_logDebug(LogType::Force, "_createNewView: Out of bounds mip level requested");
+		cemuLog_logDebug(LogType::Force, "InitAliasView(): Out of bounds mip level requested");
 		glTextureView(glTexId, glTexTarget, texture->glId_texture, glInternalFormat, texture->maxPossibleMipLevels - 1, numMip, firstSlice, this->numSlice);
 	}
 	else

src/Cafe/HW/Latte/Renderer/OpenGL/OpenGLRenderer.cpp

@@ -330,13 +330,14 @@ void OpenGLRenderer::Initialize()
 	lock.unlock();
 
 	// create framebuffer for fast clearing (avoid glClearTexSubImage on Nvidia)
-	if (this->m_vendor == GfxVendor::Nvidia || glClearTexSubImage == nullptr)
+	if (glCreateFramebuffers)
-	{
-		// generate framebuffer
-		if (glCreateFramebuffers && false)
 		glCreateFramebuffers(1, &glRendererState.clearFBO);
 	else
+	{
 		glGenFramebuffers(1, &glRendererState.clearFBO);
+		// bind to initialize
+		glBindFramebuffer(GL_FRAMEBUFFER_EXT, glRendererState.clearFBO);
+		glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
 	}
 
 	draw_init();
@@ -425,9 +426,12 @@ void _glDebugCallback(GLenum source, GLenum type, GLuint id, GLenum severity, GL
 		return;
 	if (LatteGPUState.glVendor == GLVENDOR_NVIDIA && strstr(message, "Dithering is enabled"))
 		return;
-
+	if (LatteGPUState.glVendor == GLVENDOR_NVIDIA && strstr(message, "Blending is enabled, but is not supported for integer framebuffers"))
+		return;
 	if (LatteGPUState.glVendor == GLVENDOR_NVIDIA && strstr(message, "does not have a defined base level"))
 		return;
+	if(LatteGPUState.glVendor == GLVENDOR_NVIDIA && strstr(message, "has depth comparisons disabled, with a texture object"))
+		return;
 
 	cemuLog_log(LogType::Force, "GLDEBUG: {}", message);
 
@@ -670,7 +674,10 @@ void OpenGLRenderer::rendertarget_deleteCachedFBO(LatteCachedFBO* cfbo)
 {
 	auto cfboGL = (CachedFBOGL*)cfbo;
 	if (prevBoundFBO == cfboGL->glId_fbo)
-		prevBoundFBO = -1;
+	{
+		glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+		prevBoundFBO = 0;
+	}
 	glDeleteFramebuffers(1, &cfboGL->glId_fbo);
 }
 
@@ -1013,9 +1020,6 @@ void OpenGLRenderer::texture_reserveTextureOnGPU(LatteTexture* hostTextureGeneri
 		effectiveBaseHeight = hostTexture->overwriteInfo.height;
 		effectiveBaseDepth = hostTexture->overwriteInfo.depth;
 	}
-	// get format info
-	LatteTextureGL::FormatInfoGL glFormatInfo;
-	LatteTextureGL::GetOpenGLFormatInfo(hostTexture->isDepth, hostTexture->overwriteInfo.hasFormatOverwrite ? (Latte::E_GX2SURFFMT)hostTexture->overwriteInfo.format : hostTexture->format, hostTexture->dim, &glFormatInfo);
 	// calculate mip count
 	sint32 mipLevels = std::min(hostTexture->mipLevels, hostTexture->maxPossibleMipLevels);
 	mipLevels = std::max(mipLevels, 1);
@@ -1023,25 +1027,25 @@ void OpenGLRenderer::texture_reserveTextureOnGPU(LatteTexture* hostTextureGeneri
 	if (hostTexture->dim == Latte::E_DIM::DIM_2D || hostTexture->dim == Latte::E_DIM::DIM_2D_MSAA)
 	{
 		cemu_assert_debug(effectiveBaseDepth == 1);
-		glTextureStorage2DWrapper(GL_TEXTURE_2D, hostTexture->glId_texture, mipLevels, glFormatInfo.glInternalFormat, effectiveBaseWidth, effectiveBaseHeight);
+		glTextureStorage2DWrapper(GL_TEXTURE_2D, hostTexture->glId_texture, mipLevels, hostTexture->glInternalFormat, effectiveBaseWidth, effectiveBaseHeight);
 	}
 	else if (hostTexture->dim == Latte::E_DIM::DIM_1D)
 	{
 		cemu_assert_debug(effectiveBaseHeight == 1);
 		cemu_assert_debug(effectiveBaseDepth == 1);
-		glTextureStorage1DWrapper(GL_TEXTURE_1D, hostTexture->glId_texture, mipLevels, glFormatInfo.glInternalFormat, effectiveBaseWidth);
+		glTextureStorage1DWrapper(GL_TEXTURE_1D, hostTexture->glId_texture, mipLevels, hostTexture->glInternalFormat, effectiveBaseWidth);
 	}
 	else if (hostTexture->dim == Latte::E_DIM::DIM_2D_ARRAY || hostTexture->dim == Latte::E_DIM::DIM_2D_ARRAY_MSAA)
 	{
-		glTextureStorage3DWrapper(GL_TEXTURE_2D_ARRAY, hostTexture->glId_texture, mipLevels, glFormatInfo.glInternalFormat, effectiveBaseWidth, effectiveBaseHeight, std::max(1, effectiveBaseDepth));
+		glTextureStorage3DWrapper(GL_TEXTURE_2D_ARRAY, hostTexture->glId_texture, mipLevels, hostTexture->glInternalFormat, effectiveBaseWidth, effectiveBaseHeight, std::max(1, effectiveBaseDepth));
 	}
 	else if (hostTexture->dim == Latte::E_DIM::DIM_3D)
 	{
-		glTextureStorage3DWrapper(GL_TEXTURE_3D, hostTexture->glId_texture, mipLevels, glFormatInfo.glInternalFormat, effectiveBaseWidth, effectiveBaseHeight, std::max(1, effectiveBaseDepth));
+		glTextureStorage3DWrapper(GL_TEXTURE_3D, hostTexture->glId_texture, mipLevels, hostTexture->glInternalFormat, effectiveBaseWidth, effectiveBaseHeight, std::max(1, effectiveBaseDepth));
 	}
 	else if (hostTexture->dim == Latte::E_DIM::DIM_CUBEMAP)
 	{
-		glTextureStorage3DWrapper(GL_TEXTURE_CUBE_MAP_ARRAY, hostTexture->glId_texture, mipLevels, glFormatInfo.glInternalFormat, effectiveBaseWidth, effectiveBaseHeight, effectiveBaseDepth);
+		glTextureStorage3DWrapper(GL_TEXTURE_CUBE_MAP_ARRAY, hostTexture->glId_texture, mipLevels, hostTexture->glInternalFormat, effectiveBaseWidth, effectiveBaseHeight, effectiveBaseDepth);
 	}
 	else
 	{
@@ -1279,7 +1283,6 @@ void OpenGLRenderer::texture_copyImageSubData(LatteTexture* src, sint32 srcMip,
 {
 	auto srcGL = (LatteTextureGL*)src;
 	auto dstGL = (LatteTextureGL*)dst;
-
 	if ((srcGL->isAlternativeFormat || dstGL->isAlternativeFormat) && (srcGL->glInternalFormat != dstGL->glInternalFormat))
 	{
 		if (srcGL->format == Latte::E_GX2SURFFMT::R16_G16_B16_A16_UINT && dstGL->format == Latte::E_GX2SURFFMT::BC4_UNORM)

src/Cafe/HW/Latte/Renderer/OpenGL/OpenGLRenderer.h

@@ -195,7 +195,7 @@ private:
 	GLuint glStreamoutCacheRingBuffer;
 
 	// cfbo
-	GLuint prevBoundFBO = -1;
+	GLuint prevBoundFBO = 0;
 	GLuint glId_fbo = 0;
 
 	// renderstate

src/Cafe/HW/Latte/Renderer/Vulkan/LatteTextureViewVk.cpp

@@ -57,7 +57,12 @@ uint32 LatteTextureVk_AdjustTextureCompSel(Latte::E_GX2SURFFMT format, uint32 co
 LatteTextureViewVk::LatteTextureViewVk(VkDevice device, LatteTextureVk* texture, Latte::E_DIM dim, Latte::E_GX2SURFFMT format, sint32 firstMip, sint32 mipCount, sint32 firstSlice, sint32 sliceCount)
 	: LatteTextureView(texture, firstMip, mipCount, firstSlice, sliceCount, dim, format), m_device(device)
 {
-	if (dim != texture->dim || format != texture->format)
+	if(texture->overwriteInfo.hasFormatOverwrite)
+	{
+		cemu_assert_debug(format == texture->format); // if format overwrite is used, the texture is no longer taking part in aliasing and the format of any view has to match
+		m_format = texture->GetFormat();
+	}
+	else if (dim != texture->dim || format != texture->format)
 	{
 		VulkanRenderer::FormatInfoVK texFormatInfo;
 		VulkanRenderer::GetInstance()->GetTextureFormatInfoVK(format, texture->isDepth, dim, 0, 0, &texFormatInfo);