Merge pull request #8584 from jordan-woyak/widescreen-heuristic-fix

VideoCommon: Tweak widescreen heuristic.

Source/Core/VideoCommon/RenderBase.cpp

@@ -102,7 +102,7 @@ Renderer::Renderer(int backbuffer_width, int backbuffer_height, float backbuffer
   UpdateDrawRectangle();
   CalculateTargetSize();
 
-  m_aspect_wide = SConfig::GetInstance().bWii && Config::Get(Config::SYSCONF_WIDESCREEN);
+  m_is_game_widescreen = SConfig::GetInstance().bWii && Config::Get(Config::SYSCONF_WIDESCREEN);
 }
 
 Renderer::~Renderer() = default;
@@ -561,22 +561,21 @@ void Renderer::DrawDebugText()
 
 float Renderer::CalculateDrawAspectRatio() const
 {
-  if (g_ActiveConfig.aspect_mode == AspectMode::Stretch)
-  {
-    // If stretch is enabled, we prefer the aspect ratio of the window.
+  const auto aspect_mode = g_ActiveConfig.aspect_mode;
+
+  // If stretch is enabled, we prefer the aspect ratio of the window.
+  if (aspect_mode == AspectMode::Stretch)
     return (static_cast<float>(m_backbuffer_width) / static_cast<float>(m_backbuffer_height));
+
+  const float aspect_ratio = VideoInterface::GetAspectRatio();
+
+  if (aspect_mode == AspectMode::AnalogWide ||
+      (aspect_mode == AspectMode::Auto && m_is_game_widescreen))
+  {
+    return AspectToWidescreen(aspect_ratio);
   }
 
-  // The rendering window aspect ratio as a proportion of the 4:3 or 16:9 ratio
-  if (g_ActiveConfig.aspect_mode == AspectMode::AnalogWide ||
-      (g_ActiveConfig.aspect_mode != AspectMode::Analog && m_aspect_wide))
-  {
-    return AspectToWidescreen(VideoInterface::GetAspectRatio());
-  }
-  else
-  {
-    return VideoInterface::GetAspectRatio();
-  }
+  return aspect_ratio;
 }
 
 void Renderer::AdjustRectanglesToFitBounds(MathUtil::Rectangle<int>* target_rect,
@@ -728,9 +727,7 @@ std::tuple<float, float> Renderer::ScaleToDisplayAspectRatio(const int width,
 
 void Renderer::UpdateDrawRectangle()
 {
-  // The rendering window size
-  const float win_width = static_cast<float>(m_backbuffer_width);
-  const float win_height = static_cast<float>(m_backbuffer_height);
+  const float draw_aspect_ratio = CalculateDrawAspectRatio();
 
   // Update aspect ratio hack values
   // Won't take effect until next frame
@@ -738,28 +735,10 @@ void Renderer::UpdateDrawRectangle()
   if (g_ActiveConfig.bWidescreenHack)
   {
     float source_aspect = VideoInterface::GetAspectRatio();
-    if (m_aspect_wide)
+    if (m_is_game_widescreen)
       source_aspect = AspectToWidescreen(source_aspect);
-    float target_aspect = 0.0f;
 
-    switch (g_ActiveConfig.aspect_mode)
-    {
-    case AspectMode::Stretch:
-      target_aspect = win_width / win_height;
-      break;
-    case AspectMode::Analog:
-      target_aspect = VideoInterface::GetAspectRatio();
-      break;
-    case AspectMode::AnalogWide:
-      target_aspect = AspectToWidescreen(VideoInterface::GetAspectRatio());
-      break;
-    case AspectMode::Auto:
-    default:
-      target_aspect = source_aspect;
-      break;
-    }
-
-    float adjust = source_aspect / target_aspect;
+    const float adjust = source_aspect / draw_aspect_ratio;
     if (adjust > 1)
     {
       // Vert+
@@ -775,40 +754,24 @@ void Renderer::UpdateDrawRectangle()
   }
   else
   {
-    // Hack is disabled
+    // Hack is disabled.
     g_Config.fAspectRatioHackW = 1;
     g_Config.fAspectRatioHackH = 1;
   }
 
-  // get the picture aspect ratio
-  const float draw_aspect_ratio = CalculateDrawAspectRatio();
+  // The rendering window size
+  const float win_width = static_cast<float>(m_backbuffer_width);
+  const float win_height = static_cast<float>(m_backbuffer_height);
 
   // Make ControllerInterface aware of the render window region actually being used
   // to adjust mouse cursor inputs.
   g_controller_interface.SetAspectRatioAdjustment(draw_aspect_ratio / (win_width / win_height));
 
-  float draw_width, draw_height, crop_width, crop_height;
-  draw_width = crop_width = draw_aspect_ratio;
-  draw_height = crop_height = 1;
+  float draw_width = draw_aspect_ratio;
+  float draw_height = 1;
 
-  // crop the picture to a standard aspect ratio
-  if (g_ActiveConfig.bCrop && g_ActiveConfig.aspect_mode != AspectMode::Stretch)
-  {
-    float expected_aspect = (g_ActiveConfig.aspect_mode == AspectMode::AnalogWide ||
-                             (g_ActiveConfig.aspect_mode != AspectMode::Analog && m_aspect_wide)) ?
-                                (16.0f / 9.0f) :
-                                (4.0f / 3.0f);
-    if (crop_width / crop_height >= expected_aspect)
-    {
-      // the picture is flatter than it should be
-      crop_width = crop_height * expected_aspect;
-    }
-    else
-    {
-      // the picture is skinnier than it should be
-      crop_height = crop_width / expected_aspect;
-    }
-  }
+  // Crop the picture to a standard aspect ratio. (if enabled)
+  auto [crop_width, crop_height] = ApplyStandardAspectCrop(draw_width, draw_height);
 
   // scale the picture to fit the rendering window
   if (win_width / win_height >= crop_width / crop_height)
@@ -851,6 +814,34 @@ void Renderer::SetWindowSize(int width, int height)
   Host_RequestRenderWindowSize(out_width, out_height);
 }
 
+// Crop to exactly 16:9 or 4:3 if enabled and not AspectMode::Stretch.
+std::tuple<float, float> Renderer::ApplyStandardAspectCrop(float width, float height) const
+{
+  const auto aspect_mode = g_ActiveConfig.aspect_mode;
+
+  if (!g_ActiveConfig.bCrop || aspect_mode == AspectMode::Stretch)
+    return {width, height};
+
+  // Force 4:3 or 16:9 by cropping the image.
+  const float current_aspect = width / height;
+  const float expected_aspect = (aspect_mode == AspectMode::AnalogWide ||
+                                 (aspect_mode == AspectMode::Auto && m_is_game_widescreen)) ?
+                                    (16.0f / 9.0f) :
+                                    (4.0f / 3.0f);
+  if (current_aspect > expected_aspect)
+  {
+    // keep height, crop width
+    width = height * expected_aspect;
+  }
+  else
+  {
+    // keep width, crop height
+    height = width / expected_aspect;
+  }
+
+  return {width, height};
+}
+
 std::tuple<int, int> Renderer::CalculateOutputDimensions(int width, int height) const
 {
   width = std::max(width, 1);
@@ -858,25 +849,8 @@ std::tuple<int, int> Renderer::CalculateOutputDimensions(int width, int height)
 
   auto [scaled_width, scaled_height] = ScaleToDisplayAspectRatio(width, height);
 
-  if (g_ActiveConfig.bCrop)
-  {
-    // Force 4:3 or 16:9 by cropping the image.
-    float current_aspect = scaled_width / scaled_height;
-    float expected_aspect = (g_ActiveConfig.aspect_mode == AspectMode::AnalogWide ||
-                             (g_ActiveConfig.aspect_mode != AspectMode::Analog && m_aspect_wide)) ?
-                                (16.0f / 9.0f) :
-                                (4.0f / 3.0f);
-    if (current_aspect > expected_aspect)
-    {
-      // keep height, crop width
-      scaled_width = scaled_height * expected_aspect;
-    }
-    else
-    {
-      // keep width, crop height
-      scaled_height = scaled_width / expected_aspect;
-    }
-  }
+  // Apply crop if enabled.
+  std::tie(scaled_width, scaled_height) = ApplyStandardAspectCrop(scaled_width, scaled_height);
 
   width = static_cast<int>(std::ceil(scaled_width));
   height = static_cast<int>(std::ceil(scaled_height));
@@ -1163,30 +1137,78 @@ void Renderer::EndUIFrame()
   BeginImGuiFrame();
 }
 
+// Heuristic to detect if a GameCube game is in 16:9 anamorphic widescreen mode.
+void Renderer::UpdateWidescreenHeuristic()
+{
+  // VertexManager maintains no statistics in Wii mode.
+  if (SConfig::GetInstance().bWii)
+    return;
+
+  const auto flush_statistics = g_vertex_manager->ResetFlushAspectRatioCount();
+
+  // If suggested_aspect_mode (GameINI) is configured don't use heuristic.
+  if (g_ActiveConfig.suggested_aspect_mode != AspectMode::Auto)
+    return;
+
+  // If widescreen hack isn't active and aspect_mode (UI) is 4:3 or 16:9 don't use heuristic.
+  if (!g_ActiveConfig.bWidescreenHack && (g_ActiveConfig.aspect_mode == AspectMode::Analog ||
+                                          g_ActiveConfig.aspect_mode == AspectMode::AnalogWide))
+    return;
+
+  // Modify the threshold based on which aspect ratio we're already using:
+  // If the game's in 4:3, it probably won't switch to anamorphic, and vice-versa.
+  static constexpr u32 TRANSITION_THRESHOLD = 3;
+
+  const auto looks_normal = [](auto& counts) {
+    return counts.normal_vertex_count > counts.anamorphic_vertex_count * TRANSITION_THRESHOLD;
+  };
+  const auto looks_anamorphic = [](auto& counts) {
+    return counts.anamorphic_vertex_count > counts.normal_vertex_count * TRANSITION_THRESHOLD;
+  };
+
+  const auto& persp = flush_statistics.perspective;
+  const auto& ortho = flush_statistics.orthographic;
+
+  const auto ortho_looks_anamorphic = looks_anamorphic(ortho);
+
+  if (looks_anamorphic(persp) || ortho_looks_anamorphic)
+  {
+    // If either perspective or orthographic projections look anamorphic, it's a safe bet.
+    m_is_game_widescreen = true;
+  }
+  else if (looks_normal(persp) || (m_was_orthographically_anamorphic && looks_normal(ortho)))
+  {
+    // Many widescreen games (or AR/GeckoCodes) use anamorphic perspective projections
+    // with NON-anamorphic orthographic projections.
+    // This can cause incorrect changes to 4:3 when perspective projections are temporarily not
+    // shown. e.g. Animal Crossing's inventory menu.
+    // Unless we were in a sitation which was orthographically anamorphic
+    // we won't consider orthographic data for changes from 16:9 to 4:3.
+    m_is_game_widescreen = false;
+  }
+
+  m_was_orthographically_anamorphic = ortho_looks_anamorphic;
+}
+
 void Renderer::Swap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height, u64 ticks)
 {
-  const AspectMode suggested = g_ActiveConfig.suggested_aspect_mode;
-  if (suggested == AspectMode::Analog || suggested == AspectMode::AnalogWide)
-  {
-    m_aspect_wide = suggested == AspectMode::AnalogWide;
-  }
-  else if (SConfig::GetInstance().bWii)
-  {
-    m_aspect_wide = Config::Get(Config::SYSCONF_WIDESCREEN);
-  }
-  else
-  {
-    // Heuristic to detect if a GameCube game is in 16:9 anamorphic widescreen mode.
-    const auto [flush_count_4_3, flush_count_anamorphic] =
-        g_vertex_manager->ResetFlushAspectRatioCount();
-    const size_t flush_total = flush_count_4_3 + flush_count_anamorphic;
+  if (SConfig::GetInstance().bWii)
+    m_is_game_widescreen = Config::Get(Config::SYSCONF_WIDESCREEN);
 
-    // Modify the threshold based on which aspect ratio we're already using: if
-    // the game's in 4:3, it probably won't switch to anamorphic, and vice-versa.
-    if (m_aspect_wide)
-      m_aspect_wide = !(flush_count_4_3 > 0.75 * flush_total);
-    else
-      m_aspect_wide = flush_count_anamorphic > 0.75 * flush_total;
+  // suggested_aspect_mode overrides SYSCONF_WIDESCREEN
+  if (g_ActiveConfig.suggested_aspect_mode == AspectMode::Analog)
+    m_is_game_widescreen = false;
+  else if (g_ActiveConfig.suggested_aspect_mode == AspectMode::AnalogWide)
+    m_is_game_widescreen = true;
+
+  // If widescreen hack is disabled override game's AR if UI is set to 4:3 or 16:9.
+  if (!g_ActiveConfig.bWidescreenHack)
+  {
+    const auto aspect_mode = g_ActiveConfig.aspect_mode;
+    if (aspect_mode == AspectMode::Analog)
+      m_is_game_widescreen = false;
+    else if (aspect_mode == AspectMode::AnalogWide)
+      m_is_game_widescreen = true;
   }
 
   // Ensure the last frame was written to the dump.
@@ -1225,6 +1247,10 @@ void Renderer::Swap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height, u6
       if (!IsHeadless())
       {
         BindBackbuffer({{0.0f, 0.0f, 0.0f, 1.0f}});
+
+        if (!is_duplicate_frame)
+          UpdateWidescreenHeuristic();
+
         UpdateDrawRectangle();
 
         // Adjust the source rectangle instead of using an oversized viewport to render the XFB.
@@ -1659,7 +1685,7 @@ bool Renderer::UseVertexDepthRange() const
 
 void Renderer::DoState(PointerWrap& p)
 {
-  p.Do(m_aspect_wide);
+  p.Do(m_is_game_widescreen);
   p.Do(m_frame_count);
   p.Do(m_prev_efb_format);
   p.Do(m_last_xfb_ticks);
@@ -1673,6 +1699,8 @@ void Renderer::DoState(PointerWrap& p)
     // Force the next xfb to be displayed.
     m_last_xfb_id = std::numeric_limits<u64>::max();
 
+    m_was_orthographically_anamorphic = false;
+
     // And actually display it.
     Swap(m_last_xfb_addr, m_last_xfb_width, m_last_xfb_stride, m_last_xfb_height, m_last_xfb_ticks);
   }

Source/Core/VideoCommon/RenderBase.h

@@ -50,11 +50,12 @@ enum class ShaderStage;
 enum class EFBAccessType;
 enum class EFBReinterpretType;
 enum class StagingTextureType;
+enum class AspectMode;
 
 namespace VideoCommon
 {
 class PostProcessing;
-}
+}  // namespace VideoCommon
 
 struct EfbPokeData
 {
@@ -181,6 +182,8 @@ public:
   std::tuple<float, float> ScaleToDisplayAspectRatio(int width, int height) const;
   void UpdateDrawRectangle();
 
+  std::tuple<float, float> ApplyStandardAspectCrop(float width, float height) const;
+
   // Use this to convert a single target rectangle to two stereo rectangles
   std::tuple<MathUtil::Rectangle<int>, MathUtil::Rectangle<int>>
   ConvertStereoRectangle(const MathUtil::Rectangle<int>& rc) const;
@@ -218,6 +221,8 @@ public:
   // Finish up the current frame, print some stats
   void Swap(u32 xfb_addr, u32 fb_width, u32 fb_stride, u32 fb_height, u64 ticks);
 
+  void UpdateWidescreenHeuristic();
+
   // Draws the specified XFB buffer to the screen, performing any post-processing.
   // Assumes that the backbuffer has already been bound and cleared.
   virtual void RenderXFBToScreen(const MathUtil::Rectangle<int>& target_rc,
@@ -300,7 +305,9 @@ protected:
   Common::Event m_screenshot_completed;
   std::mutex m_screenshot_lock;
   std::string m_screenshot_name;
-  bool m_aspect_wide = false;
+
+  bool m_is_game_widescreen = false;
+  bool m_was_orthographically_anamorphic = false;
 
   // The framebuffer size
   int m_target_width = 1;

Source/Core/VideoCommon/VertexManagerBase.cpp

@@ -67,16 +67,26 @@ constexpr std::array<PrimitiveType, 8> primitive_from_gx_pr{{
 // by ~9% in opposite directions.
 // Just in case any game decides to take this into account, we do both these
 // tests with a large amount of slop.
-static bool AspectIs4_3(float width, float height)
+static constexpr float ASPECT_RATIO_SLOP = 0.11f;
+
+static bool IsAnamorphicProjection(const Projection::Raw& projection, const Viewport& viewport)
 {
-  float aspect = fabsf(width / height);
-  return fabsf(aspect - 4.0f / 3.0f) < 4.0f / 3.0f * 0.11;  // within 11% of 4:3
+  // If ratio between our projection and viewport aspect ratios is similar to 16:9 / 4:3
+  // we have an anamorphic projection.
+  static constexpr float IDEAL_RATIO = (16 / 9.f) / (4 / 3.f);
+
+  const float projection_ar = projection[2] / projection[0];
+  const float viewport_ar = viewport.wd / viewport.ht;
+
+  return std::abs(std::abs(projection_ar / viewport_ar) - IDEAL_RATIO) <
+         IDEAL_RATIO * ASPECT_RATIO_SLOP;
 }
 
-static bool AspectIs16_9(float width, float height)
+static bool IsNormalProjection(const Projection::Raw& projection, const Viewport& viewport)
 {
-  float aspect = fabsf(width / height);
-  return fabsf(aspect - 16.0f / 9.0f) < 16.0f / 9.0f * 0.11;  // within 11% of 16:9
+  const float projection_ar = projection[2] / projection[0];
+  const float viewport_ar = viewport.wd / viewport.ht;
+  return std::abs(std::abs(projection_ar / viewport_ar) - 1) < ASPECT_RATIO_SLOP;
 }
 
 VertexManagerBase::VertexManagerBase()
@@ -227,12 +237,11 @@ u32 VertexManagerBase::GetRemainingIndices(int primitive) const
   }
 }
 
-std::pair<size_t, size_t> VertexManagerBase::ResetFlushAspectRatioCount()
+auto VertexManagerBase::ResetFlushAspectRatioCount() -> FlushStatistics
 {
-  std::pair<size_t, size_t> val = std::make_pair(m_flush_count_4_3, m_flush_count_anamorphic);
-  m_flush_count_4_3 = 0;
-  m_flush_count_anamorphic = 0;
-  return val;
+  const auto result = m_flush_statistics;
+  m_flush_statistics = {};
+  return result;
 }
 
 void VertexManagerBase::ResetBuffer(u32 vertex_stride)
@@ -387,18 +396,25 @@ void VertexManagerBase::Flush()
   // Track some stats used elsewhere by the anamorphic widescreen heuristic.
   if (!SConfig::GetInstance().bWii)
   {
-    const auto& raw_projection = xfmem.projection.rawProjection;
-    const bool viewport_is_4_3 = AspectIs4_3(xfmem.viewport.wd, xfmem.viewport.ht);
-    if (AspectIs16_9(raw_projection[2], raw_projection[0]) && viewport_is_4_3)
+    const bool is_perspective = xfmem.projection.type == GX_PERSPECTIVE;
+
+    auto& counts =
+        is_perspective ? m_flush_statistics.perspective : m_flush_statistics.orthographic;
+
+    if (IsAnamorphicProjection(xfmem.projection.rawProjection, xfmem.viewport))
     {
-      // Projection is 16:9 and viewport is 4:3, we are rendering an anamorphic
-      // widescreen picture.
-      m_flush_count_anamorphic++;
+      ++counts.anamorphic_flush_count;
+      counts.anamorphic_vertex_count += m_index_generator.GetIndexLen();
     }
-    else if (AspectIs4_3(raw_projection[2], raw_projection[0]) && viewport_is_4_3)
+    else if (IsNormalProjection(xfmem.projection.rawProjection, xfmem.viewport))
     {
-      // Projection and viewports are both 4:3, we are rendering a normal image.
-      m_flush_count_4_3++;
+      ++counts.normal_flush_count;
+      counts.normal_vertex_count += m_index_generator.GetIndexLen();
+    }
+    else
+    {
+      ++counts.other_flush_count;
+      counts.other_vertex_count += m_index_generator.GetIndexLen();
     }
   }
 

Source/Core/VideoCommon/VertexManagerBase.h

@@ -46,6 +46,34 @@ private:
 
   static constexpr u32 MAX_PRIMITIVES_PER_COMMAND = 65535;
 
+  // Used for 16:9 anamorphic widescreen heuristic.
+  struct FlushStatistics
+  {
+    struct ProjectionCounts
+    {
+      size_t normal_flush_count;
+      size_t anamorphic_flush_count;
+      size_t other_flush_count;
+
+      size_t normal_vertex_count;
+      size_t anamorphic_vertex_count;
+      size_t other_vertex_count;
+
+      size_t GetTotalFlushCount() const
+      {
+        return normal_flush_count + anamorphic_flush_count + other_flush_count;
+      }
+
+      size_t GetTotalVertexCount() const
+      {
+        return normal_vertex_count + anamorphic_vertex_count + other_vertex_count;
+      }
+    };
+
+    ProjectionCounts perspective;
+    ProjectionCounts orthographic;
+  };
+
 public:
   static constexpr u32 MAXVBUFFERSIZE =
       MathUtil::NextPowerOf2(MAX_PRIMITIVES_PER_COMMAND * LARGEST_POSSIBLE_VERTEX);
@@ -74,7 +102,7 @@ public:
 
   void DoState(PointerWrap& p);
 
-  std::pair<size_t, size_t> ResetFlushAspectRatioCount();
+  FlushStatistics ResetFlushAspectRatioCount();
 
   // State setters, called from register update functions.
   void SetRasterizationStateChanged() { m_rasterization_state_changed = true; }
@@ -171,8 +199,7 @@ private:
   void UpdatePipelineObject();
 
   bool m_is_flushed = true;
-  size_t m_flush_count_4_3 = 0;
-  size_t m_flush_count_anamorphic = 0;
+  FlushStatistics m_flush_statistics = {};
 
   // CPU access tracking
   u32 m_draw_counter = 0;