Fix various compiler warnings and update OpenSSL hash functions (#119)

* Update OpenSSL hash functions to OpenSSL 3.0
* Fix invalid sscanf format in DownloadManager
* Fix unset return value warning
* Fix erroneous check on otpMem in iosu_crypto

src/Cafe/Account/Account.cpp

@@ -562,9 +562,9 @@ void makePWHash(uint8* input, sint32 length, uint32 magic, uint8* output)
 	buffer[2] = (magic >> 16) & 0xFF;
 	buffer[3] = (magic >> 24) & 0xFF;
 	memcpy(buffer + 8, input, length);
-	uint8 md[32];
+	uint8 md[SHA256_DIGEST_LENGTH];
 	SHA256(buffer, 8 + length, md);
-	memcpy(output, md, 32);
+	memcpy(output, md, SHA256_DIGEST_LENGTH);
 }
 
 void actPwTest()

src/Cafe/Filesystem/FST/FST.cpp

@@ -3,7 +3,8 @@
 #include "Cemu/ncrypto/ncrypto.h"
 #include "Cafe/Filesystem/WUD/wud.h"
 #include "util/crypto/aes128.h"
-#include "openssl/sha.h"
+#include "openssl/evp.h" /* EVP_Digest */
+#include "openssl/sha.h" /* SHA1 / SHA256_DIGEST_LENGTH */
 #include "fstUtil.h"
 
 #include "FST.h"
@@ -1021,12 +1022,11 @@ bool FSTVerifier::VerifyContentFile(FileStream* fileContent, const NCrypto::AesK
 	iv[1] = (contentIndex >> 0) & 0xFF;
 	// raw content
 	uint64 remainingBytes = contentSize;
-	SHA_CTX sha1Ctx;
-	SHA256_CTX sha256Ctx;
-	if (isSHA1)
-		SHA1_Init(&sha1Ctx);
-	else
-		SHA256_Init(&sha256Ctx);
+	uint8 calculatedHash[SHA256_DIGEST_LENGTH];
+
+	EVP_MD_CTX *ctx = EVP_MD_CTX_new();
+	EVP_DigestInit(ctx, isSHA1 ? EVP_sha1() : EVP_sha256());
+
 	while (remainingBytes > 0)
 	{
 		uint32 bytesToRead = (uint32)std::min(remainingBytes, (uint64)buffer.size());
@@ -1035,26 +1035,13 @@ bool FSTVerifier::VerifyContentFile(FileStream* fileContent, const NCrypto::AesK
 		if (bytesRead != bytesToReadPadded)
 			return false;
 		AES128_CBC_decrypt_updateIV(buffer.data(), buffer.data(), bytesToReadPadded, key->b, iv);
-		if (isSHA1)
-			SHA1_Update(&sha1Ctx, buffer.data(), bytesToRead);
-		else
-			SHA256_Update(&sha256Ctx, buffer.data(), bytesToRead);
+		EVP_DigestUpdate(ctx, buffer.data(), bytesToRead);
 		remainingBytes -= bytesToRead;
 	}
-	uint8 calculatedHash[32];
-	if (isSHA1)
-		SHA1_Final(calculatedHash, &sha1Ctx);
-	else
-		SHA256_Final(calculatedHash, &sha256Ctx);
-	return memcmp(calculatedHash, tmdContentHash, isSHA1 ? 20 : 32) == 0;
-}
-
-void _SHA1Hash(void* data, size_t length, NCrypto::CHash160& hashOut)
-{
-	SHA_CTX sha1Ctx;
-	SHA1_Init(&sha1Ctx);
-	SHA1_Update(&sha1Ctx, data, length);
-	SHA1_Final(hashOut.b, &sha1Ctx);
+	unsigned int md_len;
+	EVP_DigestFinal_ex(ctx, calculatedHash, &md_len);
+	EVP_MD_CTX_free(ctx);
+	return memcmp(calculatedHash, tmdContentHash, md_len) == 0;
 }
 
 bool FSTVerifier::VerifyHashedContentFile(FileStream* fileContent, const NCrypto::AesKey* key, uint32 contentIndex, uint32 contentSize, uint32 contentSizePadded, bool isSHA1, const uint8* tmdContentHash)
@@ -1083,10 +1070,10 @@ bool FSTVerifier::VerifyHashedContentFile(FileStream* fileContent, const NCrypto
 
 		// generate H0 hash and compare
 		NCrypto::CHash160 h0;
-		_SHA1Hash(block.getFileData(), BLOCK_FILE_SIZE, h0);
-		if (memcmp(h0.b, block.getH0Hash(h0Index & 0xF), 20) != 0)
+		SHA1(block.getFileData(), BLOCK_FILE_SIZE, h0.b);
+		if (memcmp(h0.b, block.getH0Hash(h0Index & 0xF), sizeof(h0.b)) != 0)
 			return false;
-		std::memcpy(h0List[h0Index].b, h0.b, 20);
+		std::memcpy(h0List[h0Index].b, h0.b, sizeof(h0.b));
 
 		// Sixteen H0 hashes become one H1 hash
 		if (((h0Index + 1) % 16) == 0 && h0Index > 0)
@@ -1094,8 +1081,8 @@ bool FSTVerifier::VerifyHashedContentFile(FileStream* fileContent, const NCrypto
 			uint32 h1Index = ((h0Index - 15) / 16);
 
 			NCrypto::CHash160 h1;
-			_SHA1Hash(h0List.data() + h1Index * 16, sizeof(NCrypto::CHash160) * 16, h1);
-			if (memcmp(h1.b, block.getH1Hash(h1Index&0xF), 20) != 0)
+			SHA1((unsigned char *) (h0List.data() + h1Index * 16), sizeof(NCrypto::CHash160) * 16, h1.b);
+			if (memcmp(h1.b, block.getH1Hash(h1Index&0xF), sizeof(h1.b)) != 0)
 				return false;
 		}
 		// todo - repeat same for H1 and H2

src/Cafe/HW/Latte/Core/FetchShader.cpp

@@ -10,7 +10,8 @@
 #include "Cafe/HW/Latte/ISA/LatteInstructions.h"
 #include "util/containers/LookupTableL3.h"
 #include "util/helpers/fspinlock.h"
-#include <openssl/sha.h>
+#include <openssl/sha.h> /* SHA1_DIGEST_LENGTH */
+#include <openssl/evp.h> /* EVP_Digest */
 
 uint32 LatteShaderRecompiler_getAttributeSize(LatteParsedFetchShaderAttribute_t* attrib)
 {
@@ -122,20 +123,22 @@ uint32 LatteParsedFetchShaderBufferGroup_t::getCurrentBufferStride(uint32* conte
 void LatteFetchShader::CalculateFetchShaderVkHash()
 {
 	// calculate SHA1 of all states that are part of the Vulkan graphics pipeline
-	SHA_CTX ctx;
-	SHA1_Init(&ctx);
+	EVP_MD_CTX *ctx = EVP_MD_CTX_new();
+	EVP_DigestInit(ctx, EVP_sha1());
 	for(auto& group : bufferGroups)
 	{
 		// offsets
 		for (sint32 t = 0; t < group.attribCount; t++)
 		{
 			uint32 offset = group.attrib[t].offset;
-			SHA1_Update(&ctx, &t, sizeof(t));
-			SHA1_Update(&ctx, &offset, sizeof(offset));
+			EVP_DigestUpdate(ctx, &t, sizeof(t));
+			EVP_DigestUpdate(ctx, &offset, sizeof(offset));
 		}
 	}
-	uint8 shaDigest[20];
-	SHA1_Final(shaDigest, &ctx);
+	uint8 shaDigest[SHA_DIGEST_LENGTH];
+	EVP_DigestFinal_ex(ctx, shaDigest, NULL);
+	EVP_MD_CTX_free(ctx);
+
 	// fold SHA1 hash into a 64bit value
 	uint64 h = *(uint64*)(shaDigest + 0);
 	h += *(uint64*)(shaDigest + 8);

src/Cafe/HW/Latte/Renderer/Vulkan/VKRBase.h

@@ -50,6 +50,7 @@ public:
 	VKRMoveableRefCounter& operator=(VKRMoveableRefCounter&& rhs) noexcept
 	{
 		cemu_assert(false);
+		return *this;
 	}
 
 	void addRef(VKRMoveableRefCounter* refTarget)

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanPipelineStableCache.cpp

@@ -420,11 +420,8 @@ void VulkanPipelineStableCache::WorkerThread()
 		SerializePipeline(memWriter, *job);
 		auto blob = memWriter.getResult();
 		// file name is derived from data hash
-		uint8 hash[256 / 8];
-		SHA256_CTX sha256;
-		SHA256_Init(&sha256);
-		SHA256_Update(&sha256, blob.data(), blob.size());
-		SHA256_Final(hash, &sha256);
+		uint8 hash[SHA256_DIGEST_LENGTH];
+		SHA256(blob.data(), blob.size(), hash);
 		uint64 nameA = *(uint64be*)(hash + 0);
 		uint64 nameB = *(uint64be*)(hash + 8);
 		s_cache->AddFileAsync({ nameA, nameB }, blob.data(), blob.size());

src/Cafe/IOSU/legacy/iosu_act.cpp

@@ -7,7 +7,8 @@
 #include <algorithm>
 #include <mutex>
 
-#include "openssl/sha.h"
+#include "openssl/evp.h" /* EVP_Digest */
+#include "openssl/sha.h" /* SHA256_DIGEST_LENGTH */
 #include "Cafe/Account/Account.h"
 #include "config/ActiveSettings.h"
 #include "util/helpers/helpers.h"
@@ -557,24 +558,25 @@ int iosuAct_thread()
 					//   6 bytes from the end of UUID
 					// bytes 10-15 are used from the hash and replace the last 6 bytes of the UUID
 
-					SHA256_CTX ctx_sha256;
-					SHA256_Init(&ctx_sha256);
+					EVP_MD_CTX *ctx_sha256 = EVP_MD_CTX_new();
+					EVP_DigestInit(ctx_sha256, EVP_sha256());
 
-					uint8 tempArray[4];
 					uint32 name = (uint32)actCemuRequest->uuidName;
-					tempArray[0] = (name >> 24) & 0xFF;
-					tempArray[1] = (name >> 16) & 0xFF;
-					tempArray[2] = (name >> 8) & 0xFF;
-					tempArray[3] = (name >> 0) & 0xFF;
-					SHA256_Update(&ctx_sha256, tempArray, 4);
-					tempArray[0] = 0x3A;
-					tempArray[1] = 0x27;
-					tempArray[2] = 0x5E;
-					tempArray[3] = 0x09;
-					SHA256_Update(&ctx_sha256, tempArray, 4);
-					SHA256_Update(&ctx_sha256, actCemuRequest->resultBinary.binBuffer+10, 6);
-					uint8 h[32];
-					SHA256_Final(h, &ctx_sha256);
+					uint8 tempArray[] = {
+						(name >> 24) & 0xFF,
+						(name >> 16) & 0xFF,
+						(name >> 8) & 0xFF,
+						(name >> 0) & 0xFF,
+						0x3A,
+						0x27,
+						0x5E,
+						0x09,
+					};
+					EVP_DigestUpdate(ctx_sha256, tempArray, sizeof(tempArray));
+					EVP_DigestUpdate(ctx_sha256, actCemuRequest->resultBinary.binBuffer+10, 6);
+					uint8 h[SHA256_DIGEST_LENGTH];
+					EVP_DigestFinal_ex(ctx_sha256, h, NULL);
+					EVP_MD_CTX_free(ctx_sha256);
 
 					memcpy(actCemuRequest->resultBinary.binBuffer + 0xA, h + 0xA, 6);
 				}

src/Cafe/IOSU/legacy/iosu_crypto.cpp

@@ -6,7 +6,6 @@
 #include "openssl/ec.h"
 #include "openssl/x509.h"
 #include "openssl/ssl.h"
-#include "openssl/sha.h"
 #include "openssl/ecdsa.h"
 
 #include "util/crypto/aes128.h"
@@ -54,7 +53,7 @@ bool iosuCrypto_getDeviceId(uint32* deviceId)
 {
 	uint32be deviceIdBE;
 	*deviceId = 0;
-	if (otpMem == nullptr)
+	if (!hasOtpMem)
 		return false;
 	iosuCrypto_readOtpData(&deviceIdBE, 0x87, sizeof(uint32));
 	*deviceId = (uint32)deviceIdBE;
@@ -228,7 +227,7 @@ void iosuCrypto_generateDeviceCertificate()
 {
 	static_assert(sizeof(g_wiiuDeviceCert) == 0x180);
 	memset(&g_wiiuDeviceCert, 0, sizeof(g_wiiuDeviceCert));
-	if (otpMem == nullptr)
+	if (!hasOtpMem)
 		return; // cant generate certificate without OPT
 
 	// set header based on otp security mode
@@ -293,14 +292,6 @@ void iosuCrypto_generateDeviceCertificate()
 	BN_CTX_free(context);
 }
 
-void CertECC_generateHashForSignature(CertECC_t& cert, CHash256& hashOut)
-{
-	SHA256_CTX sha256;
-	SHA256_Init(&sha256);
-	SHA256_Update(&sha256, cert.certificateSubject, 0x100);
-	SHA256_Final(hashOut.b, &sha256);
-}
-
 bool iosuCrypto_hasAllDataForLogin()
 {
 	if (hasOtpMem == false)

src/Cafe/OS/libs/nn_idbe/nn_idbe.cpp

@@ -100,13 +100,13 @@ namespace nn
 			// decrypt data
 			uint8 iv[16];
 			memcpy(iv, idbeAesIv, sizeof(iv));
-			uint8 decryptedSHA256[32];
+			uint8 decryptedSHA256[SHA256_DIGEST_LENGTH];
 			AES128_CBC_decrypt_updateIV(decryptedSHA256, iconInput->hashSHA256, sizeof(decryptedSHA256), idbeAesKeys + 16 * idbeHeader->keyIndex, iv);
 			AES128_CBC_decrypt((uint8*)iconOutput, (uint8*)&iconInput->iconData, sizeof(iconInput->iconData), idbeAesKeys + 16 * idbeHeader->keyIndex, iv);
 			// calculate and compare sha256
-			uint8 calculatedSHA256[32];
-			SHA256((const unsigned char*)iconOutput, sizeof(nnIdbeIconDataV0_t), (unsigned char*)&calculatedSHA256);
-			if (memcmp(calculatedSHA256, decryptedSHA256, 32) != 0)
+			uint8 calculatedSHA256[SHA256_DIGEST_LENGTH];
+			SHA256((const unsigned char*)iconOutput, sizeof(nnIdbeIconDataV0_t), calculatedSHA256);
+			if (memcmp(calculatedSHA256, decryptedSHA256, SHA256_DIGEST_LENGTH) != 0)
 			{
 				forceLogDebug_printf("Idbe icon has incorrect sha256 hash");
 				return false;

src/Cemu/napi/napi_idbe.cpp

@@ -124,12 +124,9 @@ namespace NAPI
 		// decrypt icon data and hash
 		_decryptIDBEAndHash(&iconDataV0, hash, keyIndex);
 		// verify hash of decrypted data
-		uint8 calcHash[32];
-		SHA256_CTX shaCtx;
-		SHA256_Init(&shaCtx);
-		SHA256_Update(&shaCtx, &iconDataV0, sizeof(IDBEIconDataV0));
-		SHA256_Final(calcHash, &shaCtx);
-		if (std::memcmp(calcHash, hash, 32) != 0)
+		uint8 calcHash[SHA256_DIGEST_LENGTH];
+		SHA256((const unsigned char*) &iconDataV0, sizeof(IDBEIconDataV0), calcHash);
+		if (std::memcmp(calcHash, hash, SHA256_DIGEST_LENGTH) != 0)
 		{
 			cemuLog_log(LogType::Force, "IDBE_Request: Hash mismatch");
 			return std::nullopt;

src/Cemu/ncrypto/ncrypto.cpp

@@ -153,20 +153,14 @@ namespace NCrypto
 
 	/* Hashing */
 
-	void GenerateHashSHA256(void* data, size_t len, CHash256& hashOut)
+	void GenerateHashSHA1(const void* data, size_t len, CHash160& hashOut)
 	{
-		SHA256_CTX sha256;
-		SHA256_Init(&sha256);
-		SHA256_Update(&sha256, data, len);
-		SHA256_Final(hashOut.b, &sha256);
+		SHA1((const unsigned char*) data, len, hashOut.b);
 	}
 
-	void GenerateHashSHA1(void* data, size_t len, CHash160& hashOut)
+	void GenerateHashSHA256(const void* data, size_t len, CHash256& hashOut)
 	{
-		SHA_CTX shaCtx;
-		SHA1_Init(&shaCtx);
-		SHA1_Update(&shaCtx, data, len);
-		SHA1_Final(hashOut.b, &shaCtx);
+		SHA256((const unsigned char*) data, len, hashOut.b);
 	}
 
 	/* Ticket */
@@ -373,7 +367,7 @@ namespace NCrypto
 		EC_POINT_free(ec_publicKey);
 
 		NCrypto::CHash160 sharedKeySHA1;
-		GenerateHashSHA1(sharedKey, sharedKeyLen, sharedKeySHA1);
+		NCrypto::GenerateHashSHA1(sharedKey, sharedKeyLen, sharedKeySHA1);
 
 		uint8 aesSharedKey[16]{};
 		std::memcpy(aesSharedKey, sharedKeySHA1.b, 16);
@@ -621,11 +615,8 @@ namespace NCrypto
 
 	bool CertECC::verifySignatureViaPubKey(ECCPubKey& signerPubKey)
 	{
-		uint8 hash[32];
-		SHA256_CTX sha256;
-		SHA256_Init(&sha256);
-		SHA256_Update(&sha256, this->issuer, 0x100);
-		SHA256_Final(hash, &sha256);
+		uint8 hash[SHA256_DIGEST_LENGTH];
+		SHA256((const unsigned char *) this->issuer, 0x100, hash);
 
 		EC_KEY* ecPubKey = signerPubKey.getPublicKey();
 		ECDSA_SIG* ecSig = this->signature.getSignature();
@@ -640,11 +631,8 @@ namespace NCrypto
 
 	void CertECC::sign(ECCPrivKey& signerPrivKey)
 	{
-		uint8 hash[32];
-		SHA256_CTX sha256;
-		SHA256_Init(&sha256);
-		SHA256_Update(&sha256, this->issuer, 0x100);
-		SHA256_Final(hash, &sha256);
+		uint8 hash[SHA256_DIGEST_LENGTH];
+		SHA256((const unsigned char *) this->issuer, 0x100, hash);
 
 		// generate signature
 		EC_KEY* ec_privKey = signerPrivKey.getPrivateKey();

src/Cemu/ncrypto/ncrypto.h

@@ -197,7 +197,7 @@ namespace NCrypto
 
 	DEFINE_ENUM_FLAG_OPERATORS(TMDParser::TMDContentFlags);
 
-	void GenerateHashSHA256(void* data, size_t len, CHash256& hashOut);
+	void GenerateHashSHA256(const void* data, size_t len, CHash256& hashOut);
 	ECCSig signHash(uint32 signerTitleIdHigh, uint32 signerTitleIdLow, uint8* hash, sint32 hashLen, CertECC& certChainOut);
 
 	uint32 GetDeviceId();

src/gui/ChecksumTool.cpp

@@ -13,7 +13,8 @@
 #include <zip.h>
 #include <curl/curl.h>
 
-#include <openssl/sha.h>
+#include <openssl/evp.h> /* EVP_Digest */
+#include <openssl/sha.h> /* SHA256_DIGEST_LENGTH */
 #include <rapidjson/document.h>
 #include <rapidjson/istreamwrapper.h>
 #include <rapidjson/ostreamwrapper.h>
@@ -694,8 +695,8 @@ void ChecksumTool::DoWork()
 		const auto wud_size = wud_getWUDSize(wud);
 		std::vector<uint8> buffer(1024 * 1024 * 8);
 
-		SHA256_CTX sha256;
-		SHA256_Init(&sha256);
+		EVP_MD_CTX *sha256 = EVP_MD_CTX_new();
+		EVP_DigestInit(sha256, EVP_sha256());
 
 		uint32 read = 0;
 		size_t offset = 0;
@@ -712,7 +713,7 @@ void ChecksumTool::DoWork()
 			offset += read;
 			size -= read;
 
-			SHA256_Update(&sha256, buffer.data(), read);
+			EVP_DigestUpdate(sha256, buffer.data(), read);
 
 			wxQueueEvent(this, new wxSetGaugeValue((int)((offset * 90) / wud_size), m_progress, m_status, wxStringFormat2(_("Reading game image: {}/{}kb"), offset / 1024, wud_size / 1024)));
 		} while (read != 0 && size > 0);
@@ -723,7 +724,8 @@ void ChecksumTool::DoWork()
 		if (!m_running.load(std::memory_order_relaxed))
 			return;
 
-		SHA256_Final(checksum.data(), &sha256);
+		EVP_DigestFinal_ex(sha256, checksum.data(), NULL);
+		EVP_MD_CTX_free(sha256);
 
 		std::stringstream str;
 		for (const auto& b : checksum)
@@ -757,10 +759,7 @@ void ChecksumTool::DoWork()
 				continue;
 			}
 
-			SHA256_CTX sha256;
-			SHA256_Init(&sha256);
-			SHA256_Update(&sha256, fileData->data(), fileData->size());
-			SHA256_Final(checksum.data(), &sha256);
+			SHA256(fileData->data(), fileData->size(), checksum.data());
 
 			std::stringstream str;
 			for (const auto& b : checksum)