Move Zfreeze code out individual backends into videoCommon

Also:
 * Implement support for per-vertex PosMatrixIndex
 * Only update zslope constant once when zfreeze is activated.
 * Added a bunch of comments.

Source/Core/VideoBackends/D3D/NativeVertexFormat.cpp

@@ -59,7 +59,7 @@ DXGI_FORMAT VarToD3D(VarType t, int size, bool integer)
 
 void D3DVertexFormat::Initialize(const PortableVertexDeclaration &_vtx_decl)
 {
-	vertex_stride = _vtx_decl.stride;
+	vtx_decl = _vtx_decl;
 	memset(m_elems, 0, sizeof(m_elems));
 	const AttributeFormat* format = &_vtx_decl.position;
 

Source/Core/VideoBackends/D3D/VertexManager.cpp

@@ -181,13 +181,6 @@ void VertexManager::vFlush(bool useDstAlpha)
 
 	PrepareDrawBuffers(stride);
 
-	if (!bpmem.genMode.zfreeze)
-		CalculateZSlope(stride);
-
-	// If cull mode is CULL_ALL, do not render these triangles
-	if (bpmem.genMode.cullmode == GenMode::CULL_ALL && current_primitive_type == PRIMITIVE_TRIANGLES)
-		return;
-
 	VertexLoaderManager::GetCurrentVertexFormat()->SetupVertexPointers();
 	g_renderer->ApplyState(useDstAlpha);
 
@@ -200,9 +193,6 @@ void VertexManager::ResetBuffer(u32 stride)
 {
 	s_pCurBufferPointer = s_pBaseBufferPointer;
 	IndexGenerator::Start(GetIndexBuffer());
-
-	if (bpmem.genMode.zfreeze)
-		PixelShaderManager::SetZSlope(ZSlope.dfdx, ZSlope.dfdy, ZSlope.f0);
 }
 
 }  // namespace

Source/Core/VideoBackends/OGL/NativeVertexFormat.cpp

@@ -58,7 +58,7 @@ static void SetPointer(u32 attrib, u32 stride, const AttributeFormat &format)
 void GLVertexFormat::Initialize(const PortableVertexDeclaration &_vtx_decl)
 {
 	this->vtx_decl = _vtx_decl;
-	vertex_stride = vtx_decl.stride;
+	u32 vertex_stride = _vtx_decl.stride;
 
 	// We will not allow vertex components causing uneven strides.
 	if (vertex_stride & 3)

Source/Core/VideoBackends/OGL/VertexManager.cpp

@@ -89,9 +89,6 @@ void VertexManager::ResetBuffer(u32 stride)
 	buffer = s_indexBuffer->Map(MAXIBUFFERSIZE * sizeof(u16));
 	IndexGenerator::Start((u16*)buffer.first);
 	s_index_offset = buffer.second;
-
-	if (bpmem.genMode.zfreeze)
-		PixelShaderManager::SetZSlope(ZSlope.dfdx, ZSlope.dfdy, ZSlope.f0);
 }
 
 void VertexManager::Draw(u32 stride)
@@ -143,13 +140,6 @@ void VertexManager::vFlush(bool useDstAlpha)
 
 	PrepareDrawBuffers(stride);
 
-	if (!bpmem.genMode.zfreeze)
-		CalculateZSlope(stride);
-
-	// If cull mode is CULL_ALL, do not render these triangles
-	if (bpmem.genMode.cullmode == GenMode::CULL_ALL && current_primitive_type == PRIMITIVE_TRIANGLES)
-		return;
-
 	// Makes sure we can actually do Dual source blending
 	bool dualSourcePossible = g_ActiveConfig.backend_info.bSupportsDualSourceBlend;
 

Source/Core/VideoBackends/OGL/VertexManager.h

@@ -13,8 +13,6 @@ namespace OGL
 {
 	class GLVertexFormat : public NativeVertexFormat
 	{
-		PortableVertexDeclaration vtx_decl;
-
 	public:
 		GLVertexFormat();
 		~GLVertexFormat();

Source/Core/VideoCommon/NativeVertexFormat.h

@@ -109,7 +109,8 @@ public:
 	virtual void Initialize(const PortableVertexDeclaration &vtx_decl) = 0;
 	virtual void SetupVertexPointers() = 0;
 
-	u32 GetVertexStride() const { return vertex_stride; }
+	u32 GetVertexStride() const { return vtx_decl.stride; }
+	PortableVertexDeclaration GetVertexDeclaration() const { return vtx_decl; }
 
 	// TODO: move this under private:
 	u32 m_components;  // VB_HAS_X. Bitmask telling what vertex components are present.
@@ -118,5 +119,5 @@ protected:
 	// Let subclasses construct.
 	NativeVertexFormat() {}
 
-	u32 vertex_stride;
+	PortableVertexDeclaration vtx_decl;
 };

Source/Core/VideoCommon/VertexManagerBase.cpp

@@ -12,6 +12,7 @@
 #include "VideoCommon/RenderBase.h"
 #include "VideoCommon/Statistics.h"
 #include "VideoCommon/TextureCacheBase.h"
+#include "VideoCommon/VertexLoaderManager.h"
 #include "VideoCommon/VertexManagerBase.h"
 #include "VideoCommon/VertexShaderManager.h"
 #include "VideoCommon/VideoConfig.h"
@@ -220,6 +221,30 @@ void VertexManager::Flush()
 	GeometryShaderManager::SetConstants();
 	PixelShaderManager::SetConstants();
 
+	// Calculate ZSlope for zfreeze
+	if (!bpmem.genMode.zfreeze)
+	{
+		// Must be done after VertexShaderManager::SetConstants()
+		CalculateZSlope(VertexLoaderManager::GetCurrentVertexFormat());
+	}
+	else if (ZSlope.dirty) // or apply any dirty ZSlopes
+	{
+		PixelShaderManager::SetZSlope(ZSlope.dfdx, ZSlope.dfdy, ZSlope.f0);
+		ZSlope.dirty = false;
+	}
+
+	// If cull mode is CULL_ALL, we shouldn't render any triangles/quads (points and lines don't get culled)
+	// vertex loader has already converted any quads into triangles, so we just check for triangles.
+	// TODO: These culled primites need to get this far through the pipeline to be used as zfreeze refrence
+	//       planes. But currently we apply excessive processing and store the vertices in buffers on the
+	//       video card, which is a waste of bandwidth.
+	if (bpmem.genMode.cullmode == GenMode::CULL_ALL && current_primitive_type == PRIMITIVE_TRIANGLES)
+	{
+		GFX_DEBUGGER_PAUSE_AT(NEXT_FLUSH, true);
+		IsFlushed = true;
+		return;
+	}
+
 	bool useDstAlpha = !g_ActiveConfig.bDstAlphaPass &&
 	                   bpmem.dstalpha.enable &&
 	                   bpmem.blendmode.alphaupdate &&
@@ -245,24 +270,34 @@ void VertexManager::DoState(PointerWrap& p)
 	g_vertex_manager->vDoState(p);
 }
 
-void VertexManager::CalculateZSlope(u32 stride)
+void VertexManager::CalculateZSlope(NativeVertexFormat *format)
 {
 	float vtx[9];
 	float out[12];
 	float viewOffset[2] = { xfmem.viewport.xOrig - bpmem.scissorOffset.x * 2,
 	                        xfmem.viewport.yOrig - bpmem.scissorOffset.y * 2};
 
+	// Global matrix ID.
+	u32 mtxIdx = g_main_cp_state.matrix_index_a.PosNormalMtxIdx;
+	PortableVertexDeclaration vert_decl = format->GetVertexDeclaration();
+	size_t posOff = vert_decl.position.offset;
+	size_t mtxOff = vert_decl.posmtx.offset;
+
 	// Lookup vertices of the last rendered triangle and software-transform them
-	// This allows us to determine the depth slope, which will be used if zfreeze
+	// This allows us to determine the depth slope, which will be used if z--freeze
 	// is enabled in the following flush.
 	for (unsigned int i = 0; i < 3; ++i)
 	{
-		u8* vtx_ptr = s_pCurBufferPointer - stride * (3 - i);
+		u8* vtx_ptr = s_pCurBufferPointer - vert_decl.stride * (3 - i);
-		vtx[0 + i * 3] = ((float*)vtx_ptr)[0];
+		vtx[0 + i * 3] = ((float*)(vtx_ptr + posOff))[0];
-		vtx[1 + i * 3] = ((float*)vtx_ptr)[1];
+		vtx[1 + i * 3] = ((float*)(vtx_ptr + posOff))[1];
-		vtx[2 + i * 3] = ((float*)vtx_ptr)[2];
+		vtx[2 + i * 3] = ((float*)(vtx_ptr + posOff))[2];
 
-		VertexShaderManager::TransformToClipSpace(&vtx[i * 3], &out[i * 4]);
+		// If this vertex format has per-vertex position matrix IDs, look it up.
+		if(vert_decl.posmtx.enable)
+			mtxIdx = *((u32*)(vtx_ptr + mtxOff));
+
+		VertexShaderManager::TransformToClipSpace(&vtx[i * 3], &out[i * 4], mtxIdx);
 
 		// Transform to Screenspace
 		float inv_w = 1.0f / out[3 + i * 4];
@@ -283,11 +318,12 @@ void VertexManager::CalculateZSlope(u32 stride)
 	float b = dx31 * DF21 + dx12 * DF31;
 	float c = -dx12 * dy31 - dx31 * -dy12;
 
-	// Stop divide by zero
+	// Sometimes we process de-generate triangles. Stop any divide by zeros
 	if (c == 0)
 		return;
 
 	ZSlope.dfdx = -a / c;
 	ZSlope.dfdy = -b / c;
 	ZSlope.f0 = out[2] - (out[0] * ZSlope.dfdx + out[1] * ZSlope.dfdy);
+	ZSlope.dirty = true;
 }

Source/Core/VideoCommon/VertexManagerBase.h

@@ -4,6 +4,7 @@
 #include "Common/CommonFuncs.h"
 #include "Common/CommonTypes.h"
 #include "VideoCommon/DataReader.h"
+#include "VideoCommon/NativeVertexFormat.h"
 
 class NativeVertexFormat;
 class PointerWrap;
@@ -19,6 +20,7 @@ struct Slope
 	float dfdx;
 	float dfdy;
 	float f0;
+	bool dirty;
 };
 
 class VertexManager
@@ -63,7 +65,7 @@ protected:
 	static u32 GetRemainingIndices(int primitive);
 
 	static Slope ZSlope;
-	static void CalculateZSlope(u32 stride);
+	static void CalculateZSlope(NativeVertexFormat *format);
 
 private:
 	static bool IsFlushed;

Source/Core/VideoCommon/VertexShaderManager.cpp

@@ -690,10 +690,12 @@ void VertexShaderManager::ResetView()
 	bProjectionChanged = true;
 }
 
-void VertexShaderManager::TransformToClipSpace(const float* data, float *out)
+void VertexShaderManager::TransformToClipSpace(const float* data, float *out, u32 MtxIdx)
 {
-	// Can we use constants.posnormalmatrix here instead?
+	const float *world_matrix = (const float *)xfmem.posMatrices + (MtxIdx & 0x3f) * 4;
-	const float *world_matrix = (const float *)xfmem.posMatrices + g_main_cp_state.matrix_index_a.PosNormalMtxIdx * 4;
+	// We use the projection matrix calculated by vertexShaderManager, because it
+	// includes any free look transformations.
+	// Make sure VertexManager::SetConstants() has been called first.
 	const float *proj_matrix = &g_fProjectionMatrix[0];
 
 	float t[3];

Source/Core/VideoCommon/VertexShaderManager.h

@@ -34,11 +34,11 @@ public:
 	static void RotateView(float x, float y);
 	static void ResetView();
 
-	// data: 3 floats representing the X, Y and Z vertex model coordinates
+	// data: 3 floats representing the X, Y and Z vertex model coordinates and the posmatrix index.
 	// out: 4 floats which will be initialized with the corresponding clip space coordinates
 	// NOTE: g_fProjectionMatrix must be up to date when this is called
 	//		(i.e. VertexShaderManager::SetConstants needs to be called before using this!)
-	static void TransformToClipSpace(const float* data, float *out);
+	static void TransformToClipSpace(const float* data, float *out, u32 mtxIdx);
 
 	static VertexShaderConstants constants;
 	static bool dirty;