From ac9079f2cad9953958112da158b2ffb6c5f2e4a8 Mon Sep 17 00:00:00 2001
From: Tillmann Karras <tilkax@gmail.com>
Date: Sun, 12 Nov 2023 05:47:22 +0000
Subject: [PATCH] VideoCommon: copy software renderer logic for blend mode
 priorities

I've not tested this on hardware, but it fixes issue 12271 (shadow
people in Deal or No Deal - Special Edition).
---
 Source/Core/VideoCommon/BPMemory.cpp    |  4 +-
 Source/Core/VideoCommon/RenderState.cpp | 72 ++++++++++++-------------
 2 files changed, 37 insertions(+), 39 deletions(-)

diff --git a/Source/Core/VideoCommon/BPMemory.cpp b/Source/Core/VideoCommon/BPMemory.cpp
index 44e120470b..67952dc1ed 100644
--- a/Source/Core/VideoCommon/BPMemory.cpp
+++ b/Source/Core/VideoCommon/BPMemory.cpp
@@ -11,8 +11,8 @@ BPMemory bpmem;
 
 bool BlendMode::UseLogicOp() const
 {
-  // Logicop bit has lowest priority.
-  if (subtract || blendenable || !logicopenable)
+  // Blending overrides the logicop bit.
+  if (blendenable || !logicopenable)
     return false;
 
   // Fast path for Kirby's Return to Dreamland, they use it with dstAlpha.
diff --git a/Source/Core/VideoCommon/RenderState.cpp b/Source/Core/VideoCommon/RenderState.cpp
index ec010756ba..89f36f2b18 100644
--- a/Source/Core/VideoCommon/RenderState.cpp
+++ b/Source/Core/VideoCommon/RenderState.cpp
@@ -120,47 +120,45 @@ void BlendingState::Generate(const BPMemory& bp)
   const bool dstalpha = bp.dstalpha.enable && alphaupdate;
   usedualsrc = true;
 
-  // The subtract bit has the highest priority
-  if (bp.blendmode.subtract)
+  if (bp.blendmode.blendenable)
   {
-    blendenable = true;
-    subtractAlpha = subtract = true;
-    srcfactoralpha = srcfactor = SrcBlendFactor::One;
-    dstfactoralpha = dstfactor = DstBlendFactor::One;
-
-    if (dstalpha)
+    if (bp.blendmode.subtract)
     {
-      subtractAlpha = false;
-      srcfactoralpha = SrcBlendFactor::One;
-      dstfactoralpha = DstBlendFactor::Zero;
+      blendenable = true;
+      subtractAlpha = subtract = true;
+      srcfactoralpha = srcfactor = SrcBlendFactor::One;
+      dstfactoralpha = dstfactor = DstBlendFactor::One;
+
+      if (dstalpha)
+      {
+        subtractAlpha = false;
+        srcfactoralpha = SrcBlendFactor::One;
+        dstfactoralpha = DstBlendFactor::Zero;
+      }
+    }
+    else
+    {
+      blendenable = true;
+      srcfactor = bp.blendmode.srcfactor;
+      dstfactor = bp.blendmode.dstfactor;
+      if (!target_has_alpha)
+      {
+        // uses ONE instead of DSTALPHA
+        srcfactor = RemoveDstAlphaUsage(srcfactor);
+        dstfactor = RemoveDstAlphaUsage(dstfactor);
+      }
+      // replaces SrcClr with SrcAlpha and DstClr with DstAlpha, it is important to
+      // use the dst function for the src factor and vice versa
+      srcfactoralpha = RemoveDstColorUsage(srcfactor);
+      dstfactoralpha = RemoveSrcColorUsage(dstfactor);
+
+      if (dstalpha)
+      {
+        srcfactoralpha = SrcBlendFactor::One;
+        dstfactoralpha = DstBlendFactor::Zero;
+      }
     }
   }
-
-  // The blendenable bit has the middle priority
-  else if (bp.blendmode.blendenable)
-  {
-    blendenable = true;
-    srcfactor = bp.blendmode.srcfactor;
-    dstfactor = bp.blendmode.dstfactor;
-    if (!target_has_alpha)
-    {
-      // uses ONE instead of DSTALPHA
-      srcfactor = RemoveDstAlphaUsage(srcfactor);
-      dstfactor = RemoveDstAlphaUsage(dstfactor);
-    }
-    // replaces SrcClr with SrcAlpha and DstClr with DstAlpha, it is important to
-    // use the dst function for the src factor and vice versa
-    srcfactoralpha = RemoveDstColorUsage(srcfactor);
-    dstfactoralpha = RemoveSrcColorUsage(dstfactor);
-
-    if (dstalpha)
-    {
-      srcfactoralpha = SrcBlendFactor::One;
-      dstfactoralpha = DstBlendFactor::Zero;
-    }
-  }
-
-  // The logicop bit has the lowest priority
   else if (bp.blendmode.logicopenable)
   {
     if (bp.blendmode.logicmode == LogicOp::NoOp)