From 8129874d11c7f7d9c514cfab6e4d1dcf54b4a80d Mon Sep 17 00:00:00 2001 From: Pokechu22 Date: Thu, 24 Feb 2022 20:17:43 -0800 Subject: [PATCH] Sanitize and use increased precision when normalizing light directions This normalization was added in 02ac5e95c84a1d9a46df1dc4102342fb653e36ee, and changed to use floats in 4bf031c0646e91b35777f1ba4e2b0328063bb666. The conversion to floats means that sometimes there is insufficient precision for the normalization process, which results in values of NaN or infinity. Performing the whole process with doubles prevents that, but games also sometimes set the values to NaN or infinity directly (possibly accidentally due to the values not being initialized due to them not being used in the current configuration?). The version of Mesa currently in use on FifoCI (20.3.5) has issues with NaN. Although this bug has been fixed (https://gitlab.freedesktop.org/mesa/mesa/-/commit/b3f3287eac066eae16dce0e47aad3229dcff8257 in 21.2.0), FifoCI is stuck with the older version. This change may or may not be incorrect, but it should result in the same behavior as already present in Dolphin, while working around the Mesa bug. --- Source/Core/VideoCommon/VertexShaderManager.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Source/Core/VideoCommon/VertexShaderManager.cpp b/Source/Core/VideoCommon/VertexShaderManager.cpp index 555819e072..ce43235977 100644 --- a/Source/Core/VideoCommon/VertexShaderManager.cpp +++ b/Source/Core/VideoCommon/VertexShaderManager.cpp @@ -172,14 +172,22 @@ void VertexShaderManager::SetConstants(const std::vector& textures) dstlight.pos[1] = light.dpos[1]; dstlight.pos[2] = light.dpos[2]; + // TODO: Hardware testing is needed to confirm that this normalization is correct + auto sanitize = [](float f) { + if (std::isnan(f)) + return 0.0f; + else if (std::isinf(f)) + return f > 0.0f ? 1.0f : -1.0f; + else + return f; + }; double norm = double(light.ddir[0]) * double(light.ddir[0]) + double(light.ddir[1]) * double(light.ddir[1]) + double(light.ddir[2]) * double(light.ddir[2]); norm = 1.0 / sqrt(norm); - float norm_float = static_cast(norm); - dstlight.dir[0] = light.ddir[0] * norm_float; - dstlight.dir[1] = light.ddir[1] * norm_float; - dstlight.dir[2] = light.ddir[2] * norm_float; + dstlight.dir[0] = sanitize(static_cast(light.ddir[0] * norm)); + dstlight.dir[1] = sanitize(static_cast(light.ddir[1] * norm)); + dstlight.dir[2] = sanitize(static_cast(light.ddir[2] * norm)); } dirty = true;