From 55dc7efaa427f442bec4fa5cdbb7ad0ecba19082 Mon Sep 17 00:00:00 2001
From: Rodolfo Osvaldo Bogado <rodolfoosvaldobogado@gmail.com>
Date: Mon, 23 Nov 2009 14:08:08 +0000
Subject: [PATCH] corrected viewport and scissor test behavior in both plugins.
 some code cleanup in d3d.

git-svn-id: https://dolphin-emu.googlecode.com/svn/trunk@4605 8ced0084-cf51-0410-be5f-012b33b47a6e
---
 Source/Core/VideoCommon/Src/Render.h          |   3 +-
 .../Plugin_VideoDX9/Src/BPFunctions.cpp       |  49 +------
 .../Plugin_VideoDX9/Src/PixelShaderCache.cpp  |  43 +++----
 Source/Plugins/Plugin_VideoDX9/Src/Render.cpp | 120 ++++++++++++++++--
 .../Plugin_VideoDX9/Src/TextureCache.cpp      |   3 +-
 .../Plugin_VideoDX9/Src/TextureConverter.cpp  |   1 -
 Source/Plugins/Plugin_VideoOGL/Src/Render.cpp |  40 ++++--
 7 files changed, 163 insertions(+), 96 deletions(-)

diff --git a/Source/Core/VideoCommon/Src/Render.h b/Source/Core/VideoCommon/Src/Render.h
index d21dbcf5d4..54bfa72133 100644
--- a/Source/Core/VideoCommon/Src/Render.h
+++ b/Source/Core/VideoCommon/Src/Render.h
@@ -54,7 +54,8 @@ public:
 	static void SetColorMask();
 	static void SetBlendMode(bool forceUpdate);
 	static bool SetScissorRect();
-
+	static void SetGenerationMode();
+	static void SetDepthMode();
 	// Live resolution change
 	static bool Allow2x();
 	static bool AllowCustom();
diff --git a/Source/Plugins/Plugin_VideoDX9/Src/BPFunctions.cpp b/Source/Plugins/Plugin_VideoDX9/Src/BPFunctions.cpp
index 86354b5f1c..33ddb44aec 100644
--- a/Source/Plugins/Plugin_VideoDX9/Src/BPFunctions.cpp
+++ b/Source/Plugins/Plugin_VideoDX9/Src/BPFunctions.cpp
@@ -33,26 +33,6 @@ const bool renderFog = false;
 
 using namespace D3D;
 
-static const D3DCULL d3dCullModes[4] = 
-{
-	D3DCULL_NONE,
-	D3DCULL_CCW,
-	D3DCULL_CW,
-	D3DCULL_CCW
-};
-
-static const D3DCMPFUNC d3dCmpFuncs[8] = 
-{
-	D3DCMP_NEVER,
-	D3DCMP_LESS,
-	D3DCMP_EQUAL,
-	D3DCMP_LESSEQUAL,
-	D3DCMP_GREATER,
-	D3DCMP_NOTEQUAL,
-	D3DCMP_GREATEREQUAL,
-	D3DCMP_ALWAYS
-};
-
 static const D3DTEXTUREFILTERTYPE d3dMipFilters[4] = 
 {
 	D3DTEXF_NONE,
@@ -79,22 +59,7 @@ void FlushPipeline()
 
 void SetGenerationMode(const BPCmd &bp)
 {
-	D3D::SetRenderState(D3DRS_CULLMODE, d3dCullModes[bpmem.genMode.cullmode]);
-
-	if (bpmem.genMode.cullmode == 3)
-	{
-		D3D::SetRenderState(D3DRS_COLORWRITEENABLE, 0);
-	}
-	else
-	{
-		DWORD write = 0;
-		if (bpmem.blendmode.alphaupdate) 
-			write = D3DCOLORWRITEENABLE_ALPHA;
-		if (bpmem.blendmode.colorupdate) 
-			write |= D3DCOLORWRITEENABLE_RED | D3DCOLORWRITEENABLE_GREEN | D3DCOLORWRITEENABLE_BLUE;
-		
-		D3D::SetRenderState(D3DRS_COLORWRITEENABLE, write);
-	}
+	Renderer::SetGenerationMode();
 }
 
 void SetScissor(const BPCmd &bp)
@@ -111,17 +76,7 @@ void SetLineWidth(const BPCmd &bp)
 
 void SetDepthMode(const BPCmd &bp)
 {
-	if (bpmem.zmode.testenable)
-	{
-		D3D::SetRenderState(D3DRS_ZENABLE, TRUE);
-		D3D::SetRenderState(D3DRS_ZWRITEENABLE, bpmem.zmode.updateenable);
-		D3D::SetRenderState(D3DRS_ZFUNC, d3dCmpFuncs[bpmem.zmode.func]);		
-	}
-	else
-	{
-		D3D::SetRenderState(D3DRS_ZENABLE, FALSE);
-		D3D::SetRenderState(D3DRS_ZWRITEENABLE, FALSE);  // ??		
-	}			
+	Renderer::SetDepthMode();			
 }
 
 void SetBlendMode(const BPCmd &bp)
diff --git a/Source/Plugins/Plugin_VideoDX9/Src/PixelShaderCache.cpp b/Source/Plugins/Plugin_VideoDX9/Src/PixelShaderCache.cpp
index 08917a7011..f622b335e0 100644
--- a/Source/Plugins/Plugin_VideoDX9/Src/PixelShaderCache.cpp
+++ b/Source/Plugins/Plugin_VideoDX9/Src/PixelShaderCache.cpp
@@ -90,8 +90,23 @@ void SetPSConstant4fv(int const_number, const float *f)
 
 void PixelShaderCache::Init()
 {
-	char pmatrixprog[1024];
-	sprintf(pmatrixprog,"uniform sampler samp0 : register(s0);\n"
+	char pprog[1024];
+	sprintf(pprog,"void main(\n"
+						"out float4 ocol0 : COLOR0,\n"
+						" in float4 incol0 : COLOR0){\n"
+						"ocol0 = incol0;\n"
+						"}\n");
+	s_ClearProgram = D3D::CompilePixelShader(pprog, (int)strlen(pprog));	
+
+	sprintf(pprog,"uniform sampler samp0 : register(s0);\n"
+						"void main(\n"
+						"out float4 ocol0 : COLOR0,\n"
+						" in float3 uv0 : TEXCOORD0){\n"
+						"ocol0 = tex2D(samp0,uv0.xy);\n"						
+						"}\n");
+	s_ColorCopyProgram = D3D::CompilePixelShader(pprog, (int)strlen(pprog));
+
+	sprintf(pprog,"uniform sampler samp0 : register(s0);\n"
 						"uniform float4 cColMatrix[5] : register(c%d);\n"
 						"void main(\n"
 						"out float4 ocol0 : COLOR0,\n"
@@ -99,23 +114,9 @@ void PixelShaderCache::Init()
 						"float4 texcol = tex2D(samp0,uv0.xy);\n"
 						"ocol0 = float4(dot(texcol,cColMatrix[0]),dot(texcol,cColMatrix[1]),dot(texcol,cColMatrix[2]),dot(texcol,cColMatrix[3])) + cColMatrix[4];\n"						
 						"}\n",C_COLORMATRIX);
-	char pcopyprog[1024];
-	sprintf(pcopyprog,"uniform sampler samp0 : register(s0);\n"
-						"void main(\n"
-						"out float4 ocol0 : COLOR0,\n"
-						" in float3 uv0 : TEXCOORD0){\n"
-						"ocol0 = tex2D(samp0,uv0.xy);\n"						
-						"}\n");
-	
-	char pclearprog[1024];
-	sprintf(pclearprog,"void main(\n"
-						"out float4 ocol0 : COLOR0,\n"
-						" in float4 incol0 : COLOR0){\n"
-						"ocol0 = incol0;\n"
-						"}\n");
+	s_ColorMatrixProgram = D3D::CompilePixelShader(pprog, (int)strlen(pprog));
 
-	char pdmatrixprog[1024];
-	sprintf(pdmatrixprog,"uniform sampler samp0 : register(s0);\n"
+	sprintf(pprog,"uniform sampler samp0 : register(s0);\n"
 						"uniform float4 cColMatrix[5] : register(c%d);\n"
 						"void main(\n"
 						"out float4 ocol0 : COLOR0,\n"
@@ -125,11 +126,7 @@ void PixelShaderCache::Init()
 						"texcol = float4((EncodedDepth.rgb * (16777216.0f/16777215.0f)),1.0f);\n"
 						"ocol0 = float4(dot(texcol,cColMatrix[0]),dot(texcol,cColMatrix[1]),dot(texcol,cColMatrix[2]),dot(texcol,cColMatrix[3])) + cColMatrix[4];\n"						
 						"}\n",C_COLORMATRIX);
-
-	s_ColorMatrixProgram = D3D::CompilePixelShader(pmatrixprog, (int)strlen(pmatrixprog));
-	s_ColorCopyProgram = D3D::CompilePixelShader(pcopyprog, (int)strlen(pcopyprog));
-	s_DepthMatrixProgram = D3D::CompilePixelShader(pdmatrixprog, (int)strlen(pdmatrixprog));
-	s_ClearProgram = D3D::CompilePixelShader(pclearprog, (int)strlen(pclearprog));	
+	s_DepthMatrixProgram = D3D::CompilePixelShader(pprog, (int)strlen(pprog));	
 	Clear();
 }
 
diff --git a/Source/Plugins/Plugin_VideoDX9/Src/Render.cpp b/Source/Plugins/Plugin_VideoDX9/Src/Render.cpp
index e5997bd588..c464866320 100644
--- a/Source/Plugins/Plugin_VideoDX9/Src/Render.cpp
+++ b/Source/Plugins/Plugin_VideoDX9/Src/Render.cpp
@@ -90,6 +90,29 @@ static const D3DBLEND d3dDestFactors[8] =
 	D3DBLEND_INVDESTALPHA
 };
 
+
+static const D3DCULL d3dCullModes[4] = 
+{
+	D3DCULL_NONE,
+	D3DCULL_CCW,
+	D3DCULL_CW,
+	D3DCULL_CCW
+};
+
+static const D3DCMPFUNC d3dCmpFuncs[8] = 
+{
+	D3DCMP_NEVER,
+	D3DCMP_LESS,
+	D3DCMP_EQUAL,
+	D3DCMP_LESSEQUAL,
+	D3DCMP_GREATER,
+	D3DCMP_NOTEQUAL,
+	D3DCMP_GREATEREQUAL,
+	D3DCMP_ALWAYS
+};
+
+
+
 void SetupDeviceObjects()
 {
 	D3D::font.Init();
@@ -463,6 +486,18 @@ bool Renderer::SetScissorRect()
 	if (rc.bottom < 0) rc.bottom = 0;
 	if (rc.top > s_target_height) rc.top = s_target_height;
 	if (rc.bottom > s_target_height) rc.bottom = s_target_height;
+	if(rc.left > rc.right)
+	{
+		int temp = rc.right;
+		rc.right = rc.left;
+		rc.left = temp;
+	}
+	if(rc.top > rc.bottom)
+	{
+		int temp = rc.bottom;
+		rc.bottom = rc.top;
+		rc.top = temp;
+	}
 	
 	if (rc.right >= rc.left && rc.bottom >= rc.top)
 	{
@@ -471,7 +506,7 @@ bool Renderer::SetScissorRect()
 	}
 	else
 	{
-		WARN_LOG(VIDEO, "Bad scissor rectangle: %i %i %i %i", rc.left, rc.top, rc.right, rc.bottom);
+		//WARN_LOG(VIDEO, "Bad scissor rectangle: %i %i %i %i", rc.left, rc.top, rc.right, rc.bottom);
 		rc.left   = 0;
 		rc.top    = 0;
 		rc.right  = GetTargetWidth();
@@ -592,7 +627,6 @@ u32 Renderer::AccessEFB(EFBAccessType type, int x, int y)
 
 		hr = D3D::dev->SetRenderTarget(0, FBManager::GetEFBColorRTSurface());
 		hr = D3D::dev->SetDepthStencilSurface(FBManager::GetEFBDepthRTSurface());
-		UpdateViewport();
 		RestoreAPIState();
 		RectToLock.bottom = 4;
 		RectToLock.left = 0;
@@ -684,6 +718,13 @@ u32 Renderer::AccessEFB(EFBAccessType type, int x, int y)
 // Called from VertexShaderManager
 void UpdateViewport()
 {
+	// reversed gxsetviewport(xorig, yorig, width, height, nearz, farz)
+    // [0] = width/2
+    // [1] = height/2
+    // [2] = 16777215 * (farz - nearz)
+    // [3] = xorig + width/2 + 342
+    // [4] = yorig + height/2 + 342
+    // [5] = 16777215 * farz
 	int scissorXOff = bpmem.scissorOffset.x * 2;
 	int scissorYOff = bpmem.scissorOffset.y * 2;
 
@@ -695,17 +736,29 @@ void UpdateViewport()
 	// Stretch picture with increased internal resolution
 	int X = (int)(ceil(xfregs.rawViewport[3] - xfregs.rawViewport[0] - (scissorXOff)) * MValueX); 	
 	int Y = (int)(ceil(xfregs.rawViewport[4] + xfregs.rawViewport[1] - (scissorYOff)) * MValueY);
-	int Width  = (int)ceil(abs((int)(2 * xfregs.rawViewport[0])) * MValueX);
-	int Height = (int)ceil(abs((int)(2 * xfregs.rawViewport[1])) * MValueY);
+	int Width  = (int)ceil((int)(2 * xfregs.rawViewport[0]) * MValueX);
+	int Height = (int)ceil((int)(-2 * xfregs.rawViewport[1]) * MValueY);
 
+	if(Width < 0)
+	{
+		X += Width;
+		Width*=-1;
+		if( X < 0)
+		{
+			Width +=X;
+		}
+	}
+	if(Height < 0)
+	{
+		Y += Height;
+		Height *= -1;
+		if(Y < 0)
+		{
+			Height+=Y;
+		}
+	}
 	if(X < 0) X = 0;
-	if(Y < 0) Y = 0;
-	if(X > s_target_width) X = s_target_width;
-	if(Y > s_target_height) Y = s_target_height;
-	if(Width < 0) Width = 0;
-	if(Height < 0) Height = 0;
-	if(Width > (s_target_width - X)) Width = s_target_width - X;
-	if(Height > (s_target_height - Y)) Height = s_target_height - Y;
+	if(Y < 0) Y = 0;	
 	vp.X = X;
 	vp.Y = Y;
 	vp.Width = Width;
@@ -736,11 +789,15 @@ void Renderer::ClearScreen(const EFBRectangle& rc, bool colorEnable, bool alphaE
 	sirc.top    = targetRc.top;
 	sirc.right  = targetRc.right;
 	sirc.bottom = targetRc.bottom;
-	D3D::dev->SetScissorRect(&sirc);
+	D3D::dev->SetScissorRect(&sirc);	
+	if(zEnable)
+		D3D::SetRenderState(D3DRS_ZFUNC, D3DCMP_ALWAYS);
 	D3D::drawClearQuad(&sirc,color,(z & 0xFFFFFF) / float(0xFFFFFF),PixelShaderCache::GetClearProgram(),VertexShaderCache::GetSimpleVertexShader());	
-	//D3D::dev->Clear(0, NULL, clearflags, color,(z & 0xFFFFFF) / float(0xFFFFFF), 0);			
+	if(zEnable)
+		D3D::SetRenderState(D3DRS_ZFUNC, d3dCmpFuncs[bpmem.zmode.func]);
+	//D3D::dev->Clear(0, NULL, (colorEnable ? D3DCLEAR_TARGET : 0)| ( zEnable ? D3DCLEAR_ZBUFFER : 0), color,(z & 0xFFFFFF) / float(0xFFFFFF), 0);			
 	SetScissorRect();
-	VertexShaderManager::SetViewportChanged();	
+	UpdateViewport();
 }
 
 void Renderer::SetBlendMode(bool forceUpdate)
@@ -840,3 +897,38 @@ void Renderer::RestoreAPIState()
     SetColorMask();
 	SetBlendMode(true);	
 }
+
+void Renderer::SetGenerationMode()
+{
+	D3D::SetRenderState(D3DRS_CULLMODE, d3dCullModes[bpmem.genMode.cullmode]);
+
+	if (bpmem.genMode.cullmode == 3)
+	{
+		D3D::SetRenderState(D3DRS_COLORWRITEENABLE, 0);
+	}
+	else
+	{
+		DWORD write = 0;
+		if (bpmem.blendmode.alphaupdate) 
+			write = D3DCOLORWRITEENABLE_ALPHA;
+		if (bpmem.blendmode.colorupdate) 
+			write |= D3DCOLORWRITEENABLE_RED | D3DCOLORWRITEENABLE_GREEN | D3DCOLORWRITEENABLE_BLUE;
+		
+		D3D::SetRenderState(D3DRS_COLORWRITEENABLE, write);
+	}
+}
+
+void Renderer::SetDepthMode()
+{
+	if (bpmem.zmode.testenable)
+	{
+		D3D::SetRenderState(D3DRS_ZENABLE, TRUE);
+		D3D::SetRenderState(D3DRS_ZWRITEENABLE, bpmem.zmode.updateenable);
+		D3D::SetRenderState(D3DRS_ZFUNC, d3dCmpFuncs[bpmem.zmode.func]);		
+	}
+	else
+	{
+		D3D::SetRenderState(D3DRS_ZENABLE, FALSE);
+		D3D::SetRenderState(D3DRS_ZWRITEENABLE, FALSE);  // ??		
+	}			
+}
diff --git a/Source/Plugins/Plugin_VideoDX9/Src/TextureCache.cpp b/Source/Plugins/Plugin_VideoDX9/Src/TextureCache.cpp
index 6140767986..a103a8c747 100644
--- a/Source/Plugins/Plugin_VideoDX9/Src/TextureCache.cpp
+++ b/Source/Plugins/Plugin_VideoDX9/Src/TextureCache.cpp
@@ -490,8 +490,7 @@ have_texture:
 	}
 
 	D3D::dev->SetRenderTarget(0, FBManager::GetEFBColorRTSurface());
-	D3D::dev->SetDepthStencilSurface(FBManager::GetEFBDepthRTSurface());
-	VertexShaderManager::SetViewportChanged();
+	D3D::dev->SetDepthStencilSurface(FBManager::GetEFBDepthRTSurface());	
 	Renderer::RestoreAPIState();	
 	Rendersurf->Release();
 }
diff --git a/Source/Plugins/Plugin_VideoDX9/Src/TextureConverter.cpp b/Source/Plugins/Plugin_VideoDX9/Src/TextureConverter.cpp
index e4c4ecdcbc..188a91aea1 100644
--- a/Source/Plugins/Plugin_VideoDX9/Src/TextureConverter.cpp
+++ b/Source/Plugins/Plugin_VideoDX9/Src/TextureConverter.cpp
@@ -230,7 +230,6 @@ void EncodeToRamUsingShader(LPDIRECT3DPIXELSHADER9 shader, LPDIRECT3DTEXTURE9 sr
 	D3D::drawShadedTexQuad(srcTexture,&SrcRect,srcTextureWidth,srcTextureHeight,&DstRect,shader,VertexShaderCache::GetSimpleVertexShader());
 	hr = D3D::dev->SetRenderTarget(0, FBManager::GetEFBColorRTSurface());
 	hr = D3D::dev->SetDepthStencilSurface(FBManager::GetEFBDepthRTSurface());
-	VertexShaderManager::SetViewportChanged();
 	Renderer::RestoreAPIState();	
 	D3D::RefreshSamplerState(0, D3DSAMP_MINFILTER);
 	D3D::RefreshSamplerState(0, D3DSAMP_MAGFILTER);
diff --git a/Source/Plugins/Plugin_VideoOGL/Src/Render.cpp b/Source/Plugins/Plugin_VideoOGL/Src/Render.cpp
index 3fe212055a..6fd6aaa9d4 100644
--- a/Source/Plugins/Plugin_VideoOGL/Src/Render.cpp
+++ b/Source/Plugins/Plugin_VideoOGL/Src/Render.cpp
@@ -700,11 +700,18 @@ bool Renderer::SetScissorRect()
 	rc_bottom *= MValueY;
 	if (rc_bottom > EFB_HEIGHT * MValueY) rc_bottom = EFB_HEIGHT * MValueY;
 
-   /*LOG(VIDEO, "Scissor: lt=(%d,%d), rb=(%d,%d,%i), off=(%d,%d)\n",
-		rc_left, rc_top,
-		rc_right, rc_bottom, Renderer::GetTargetHeight(),
-		xoff, yoff
-		);*/
+   if(rc_left > rc_right)
+	{
+		int temp = rc_right;
+		rc_right = rc_left;
+		rc_left = temp;
+	}
+	if(rc_top > rc_bottom)
+	{
+		int temp = rc_bottom;
+		rc_bottom = rc_top;
+		rc_top = temp;
+	}
 
 	// Check that the coordinates are good
     if (rc_right >= rc_left && rc_bottom >= rc_top)
@@ -1333,6 +1340,13 @@ void Renderer::FlipImageData(u8 *data, int w, int h)
 // Called from VertexShaderManager
 void UpdateViewport()
 {
+	// reversed gxsetviewport(xorig, yorig, width, height, nearz, farz)
+    // [0] = width/2
+    // [1] = height/2
+    // [2] = 16777215 * (farz - nearz)
+    // [3] = xorig + width/2 + 342
+    // [4] = yorig + height/2 + 342
+    // [5] = 16777215 * farz
 	int scissorXOff = bpmem.scissorOffset.x * 2;   // 342
 	int scissorYOff = bpmem.scissorOffset.y * 2;   // 342
 
@@ -1342,10 +1356,20 @@ void UpdateViewport()
 	// Stretch picture with increased internal resolution
 	int GLx = (int)ceil((xfregs.rawViewport[3] - xfregs.rawViewport[0] - scissorXOff) * MValueX);
 	int GLy = (int)ceil(Renderer::GetTargetHeight() - ((int)(xfregs.rawViewport[4] - xfregs.rawViewport[1] - scissorYOff)) * MValueY);
-	int GLWidth = (int)ceil(abs((int)(2 * xfregs.rawViewport[0])) * MValueX);
-	int GLHeight = (int)ceil(abs((int)(2 * xfregs.rawViewport[1])) * MValueY);
+	int GLWidth = (int)ceil((int)(2 * xfregs.rawViewport[0]) * MValueX);
+	int GLHeight = (int)ceil((int)(-2 * xfregs.rawViewport[1]) * MValueY);
 	double GLNear = (xfregs.rawViewport[5] - xfregs.rawViewport[2]) / 16777216.0f;
-	double GLFar = xfregs.rawViewport[5] / 16777216.0f;	
+	double GLFar = xfregs.rawViewport[5] / 16777216.0f;
+	if(GLWidth < 0)
+	{
+		GLx += GLWidth;
+		GLWidth*=-1;		
+	}
+	if(GLHeight < 0)
+	{
+		GLy += GLHeight;
+		GLHeight *= -1;
+	}
 	// Update the view port
 	glViewport(GLx, GLy, GLWidth, GLHeight);
 	glDepthRange(GLNear, GLFar);