cemu_graphic_packs/docs/v4-converter/verify-graphicPacks.js

const fs = require("fs");
const path = require("path");
const child_process = require("child_process");

var globalShaders = {};
var unverifiedShaders = [];
var invalidShaders = [];
var verifiedShaders = [];

const attributeLayoutRegex = /ATTR_LAYOUT\(\d+\s*,\s*(\d+)\)\s*in\s*uvec4\s*([a-zA-Z0-9_]+)\s*;/;
const bufferLayoutRegex = /UNIFORM_BUFFER_LAYOUT\((\d+)\s*,\s*\d+,\s*(\d+)\)(?:\s*uniform\s*([a-zA-Z]+\d)\s*)?/;
const splitBufferLayoutRegex = /uniform\s*([a-zA-Z]+\d+)\s*/;
const textureLayoutRegex = /TEXTURE_LAYOUT\((\d+), \d+, (\d+)\) uniform sampler2D ([a-zA-Z]+\d);/;
const ufBlocklayoutRegex = /layout\(set\s?=\s?\d+,\s?binding\s?=\s?(\d+)\)\s*uniform\s+ufBlock/;
const uniformUfRegex = /^\s*uniform\s+([biuvd]?vec[234]|bool|int|uint|float|double)\s+(uf_alphaTestRef|uf_verticesPerInstance|uf_streamoutBufferBase\[\d+\]|uf_tex\[\d+\]Scale|uf_pointSize|uf_fragCoordScale|uf_windowSpaceToClipSpaceTransform|uf_remappedPS\[\d+\]|uf_remappedVS\[\d+\]|uf_uniformRegisterVS\[\d+\]|uf_uniformRegisterPS\[\d+\])\s*;.*/m;

function extractShaderInfo(shaderText) {
	let shaderInfo = {attributeLayouts: [], bufferLayouts: [], textureLayouts: [], ufBlock: {VKLocation: undefined, ufVariables: []}};
	let shaderLines = shaderText.split("\n");
	
	let ufBlockFlag = false;
	for (let line = 0; line<shaderLines.length; line++) {
		let currLine = shaderLines[line];

		let attributeRegexResult = currLine.match(attributeLayoutRegex);
		let bufferRegexResult = currLine.match(bufferLayoutRegex);
		let textureRegexResult = currLine.match(textureLayoutRegex);
		if (attributeRegexResult != null) {
			shaderInfo.attributeLayouts.push({location: parseInt(attributeRegexResult[1]), name: attributeRegexResult[2]});
		}
		else if (bufferRegexResult != null) {
			if (bufferRegexResult[3] != undefined) {
				shaderInfo.bufferLayouts.push({GLLocation: parseInt(bufferRegexResult[1]), VKLocation: parseInt(bufferRegexResult[2]), name: bufferRegexResult[3]});
			}
			else {
				let splitBufferResult = shaderLines[line+1].match(splitBufferLayoutRegex);
				if (splitBufferResult != null) {
					shaderInfo.bufferLayouts.push({GLLocation: parseInt(bufferRegexResult[1]), VKLocation: parseInt(bufferRegexResult[2]), name: splitBufferResult[1]});
				}
				else {
					console.info(currLine.trim()+" vs "+shaderLines[line+1]);
				}
			}
		}
		else if (textureRegexResult != null) {
			shaderInfo.textureLayouts.push({GLLocation: parseInt(textureRegexResult[1]), VKLocation: parseInt(textureRegexResult[2]), name: textureRegexResult[3]});
		}

		if (ufBlocklayoutRegex.test(currLine)) {
			shaderInfo.ufBlock.VKLocation = parseInt(currLine.match(ufBlocklayoutRegex)[1]);
			ufBlockFlag = true;
		}
		else if (ufBlockFlag && currLine.includes("};")) {
			ufBlockFlag = false;
		}
		else if (ufBlockFlag) {
			let uniformMatch = currLine.match(uniformUfRegex);
			if (uniformMatch != null) {
				shaderInfo.ufBlock.ufVariables.push({type:uniformMatch[1], name: uniformMatch[2]});
			}
			//else if (!currLine.startsWith("{") && currLine.trim() == "") console.log(currLine);
		}
	}

	if (shaderInfo.ufBlock.VKLocation != undefined) {
		//console.error("This shader doesn't have a ufBlock...", shaderInfo);
	}
	return shaderInfo;
}

function getPresets(rulesText) {
	let rulesPresets = [];
	let rulesLines = rulesText.split("\n");
	let packVersion = undefined;

	let currentSection = "";
	let currentPreset = {name: "", variables: []};

	for (line in rulesLines) {
		if (rulesLines[line].replace(/\s+/g, "").startsWith("version=3")) packVersion = 3;
		if (rulesLines[line].replace(/\s+/g, "").startsWith("version=4")) packVersion = 4;

		if (/^\[(.+)]/.test(rulesLines[line])) {
			if (currentSection == "Preset") {
				rulesPresets.push({name: currentPreset.name, variables: [...currentPreset.variables]});
				currentPreset = null;
			}
			currentSection = rulesLines[line].match(/^\[(.+)]/)[1];
			if (currentSection == "Preset") {
				currentPreset = {name: "", variables: []};
			}
		}

		if (currentSection == "Preset") {
			let nameMatch = rulesLines[line].match(/^name[ ]?=[ ]?([^\n#]+)[ ]?/m);
			if (nameMatch != null) {
				currentPreset.name = nameMatch[1].trim();
			}
			let presetVariable = rulesLines[line].match(/^(?<key>\$[a-zA-Z_]+)(?:(?<isInt>:int))?[ ]*=[ ]*(?<value>\d+\.\d+|\d+|0x[0-9a-fA-F]+|(?<expression>[+\-*\/\(\)\d]+))/m);
			if (presetVariable != null) {
				if (presetVariable.groups.expression != undefined) currentPreset.variables.push({key: presetVariable.groups.key, value: (presetVariable.groups.isInt != undefined ? 666 : 666.666667), isInt: (presetVariable.groups.isInt != undefined)});
				else currentPreset.variables.push({key: presetVariable.groups.key, value: (presetVariable.groups.isInt != undefined ? parseInt(presetVariable.groups.value) : parseFloat(presetVariable.groups.value)), isInt: (presetVariable.groups.isInt != undefined)});
			}
		}
	}
	if (currentSection == "Preset") rulesPresets.push({name: currentPreset.name, variables: [...currentPreset.variables]});

	for (let i=0; i<rulesPresets.length; i++) {
		rulesPresets[i].variables.sort((a, b) => {
			return (b.key.length - a.key.length);
		});
	}

	if (rulesPresets.length == 0) rulesPresets.push({name: "Default", variables: []});
	return [rulesPresets, packVersion];
}

function verifyShader(rulesPresets, dumpShader, shaderText, vulkanSet, shaderPath) {
	// First, remove the Vulkan header
	let shaderLines = shaderText.split("\n");
	let parsingVulkanHeader = false;
	for (line in shaderLines) {
		if (shaderLines[line].trim().startsWith("#ifdef VULKAN")) parsingVulkanHeader = true;
		else if (parsingVulkanHeader && (shaderLines[line].trim().startsWith("#define") || shaderLines[line].trim().startsWith("#else"))) shaderLines[line] = "\r";
		else if (parsingVulkanHeader && shaderLines[line].trim().startsWith("#endif")) break;
		//else if (parsingVulkanHeader) console.error("what's this", shaderLines[line]);
	}

	// Replace presets
	for (preset in rulesPresets) {
		let currPreset = rulesPresets[preset];
		let headerlessPresetLines = [];

		for (line in shaderLines) {
			let currLine = shaderLines[line];

			for (presetVar in currPreset.variables) {
				let currPresetVar = currPreset.variables[presetVar];
				let currPresetVarOccur = currLine.split(currPresetVar.key).length-1;
				for (let i=0; i<currPresetVarOccur; i++) {
					if (currPresetVar.isInt) currLine = currLine.replace(currPresetVar.key, currPresetVar.value.toFixed(0));
					else currLine = currLine.replace(currPresetVar.key, currPresetVar.value.toFixed(3));
				}
			}

			headerlessPresetLines.push(currLine);
		}

		// Remove all the unrelated preset code
		let preprocessedPresetLines = undefined;
		try {
			let preprocessOutput = child_process.spawnSync("glslangValidator.exe", ["--stdin", "-E", "-DVULKAN", "-S "+((vulkanSet == 0) ? "vert" : "frag")], {encoding: "utf8", shell: true, input: headerlessPresetLines.join("\n")});
			if (preprocessOutput.stderr.trim() != "") throw preprocessOutput.stderr;
			preprocessedPresetLines = preprocessOutput.stdout.split("\n");
		}
		catch (err) {
			console.error("An error occured while preprocessing this shader:\r\n", err);
			continue;
		}

		let shaderInfo = extractShaderInfo(preprocessedPresetLines.join("\n"));
		if (shaderInfo.ufBlock.VKLocation == undefined) {
			//console.error(preprocessedPresetLines);
		}

		// Check the dumped shader for matching bindings
		let attributesMatched = true;
		for (attr in shaderInfo.attributeLayouts) {
			let foundMatch = false;
			for (cmpAttr in dumpShader.attributeLayouts) {
				if (dumpShader.attributeLayouts[cmpAttr].name == shaderInfo.attributeLayouts[attr].name && dumpShader.attributeLayouts[cmpAttr].location == shaderInfo.attributeLayouts[attr].location) foundMatch = true;
			}
			if (!foundMatch) attributesMatched = false;
		}
		if (!attributesMatched) console.error("The attributes didn't match.");
		let bufferLayoutsMatched = true;
		for (buffer in shaderInfo.bufferLayouts) {
			let foundMatch = false;
			for (cmpBuffer in dumpShader.bufferLayouts) {
				if (dumpShader.bufferLayouts[cmpBuffer].name == shaderInfo.bufferLayouts[buffer].name && dumpShader.bufferLayouts[cmpBuffer].VKLocation == shaderInfo.bufferLayouts[buffer].VKLocation && dumpShader.bufferLayouts[cmpBuffer].GLLocation == shaderInfo.bufferLayouts[buffer].GLLocation) foundMatch = true;
			}
			if (!foundMatch) bufferLayoutsMatched = false;
		}
		if (!bufferLayoutsMatched) console.error("The buffer layouts didn't match.");
		let textureLayoutsMatched = true;
		for (texture in shaderInfo.textureLayouts) {
			let foundMatch = false;
			for (cmpTexture in dumpShader.textureLayouts) {
				if (dumpShader.textureLayouts[cmpTexture].name == shaderInfo.textureLayouts[texture].name && dumpShader.textureLayouts[cmpTexture].VKLocation == shaderInfo.textureLayouts[texture].VKLocation && dumpShader.textureLayouts[cmpTexture].GLLocation == shaderInfo.textureLayouts[texture].GLLocation) foundMatch = true;
			}
			if (!foundMatch) textureLayoutsMatched = false;
		}
		if (!textureLayoutsMatched) console.error("The texture layouts didn't match.");

		// Check if ufBlock matches
		let ufBlockLocationMatched = true;
		if (shaderInfo.ufBlock.VKLocation != dumpShader.ufBlock.VKLocation) {
			console.error("The ufBlock location differs...");
			ufBlockLocationMatched = false;
		}
		let ufVariableMismatches = [];
		if (shaderInfo.ufBlock.ufVariables.length == dumpShader.ufBlock.ufVariables.length) {
			for (let i=0; i<shaderInfo.ufBlock.ufVariables.length; i++) {
				if (shaderInfo.ufBlock.ufVariables[i].type != dumpShader.ufBlock.ufVariables[i].type || shaderInfo.ufBlock.ufVariables[i].name != dumpShader.ufBlock.ufVariables[i].name) ufVariableMismatches.push(shaderInfo.ufBlock.ufVariables[i]);
			}
		}
		else {
			console.error("Shader didn't have the same amount of uf_* variables:\r\nshader: "+JSON.stringify(shaderInfo.ufBlock)+"\r\ndump: "+JSON.stringify(dumpShader.ufBlock));
			ufBlockMatched = false;
		}
		if (ufVariableMismatches.length != 0) console.error("The uf_* variables didn't match!", ufVariableMismatches);

		if (attributesMatched && bufferLayoutsMatched && textureLayoutsMatched && ufBlockLocationMatched && (ufVariableMismatches.length == 0)) {
			//console.info("The shader matched!");
			if (!invalidShaders.includes(shaderPath) && !verifiedShaders.includes(shaderPath)) verifiedShaders.push(shaderPath);
		}
		else {
			console.error("The shader didn't match... please fix the errors above.");
			if (!invalidShaders.includes(shaderPath)) invalidShaders.push(shaderPath);
			let verifiedRemoveIndex = verifiedShaders.indexOf(shaderPath);
			if (verifiedRemoveIndex != -1) verifiedShaders.slice(verifiedRemoveIndex, 1);
		}
	}
}

function verifyPack(analyseFiles, folderArray) {
	if (analyseFiles.shaders.length == 0) return;
	
	let [rulesPresets, packVersion] = getPresets(analyseFiles.rulesText);

	for (shader in analyseFiles.shaders) {
		if (!globalShaders.hasOwnProperty(path.basename(analyseFiles.shaders[shader].name, ".txt"))) {
			console.error("Can't validate this "+path.basename(analyseFiles.shaders[shader].name)+" shader.");
			unverifiedShaders.push(path.join(process.cwd(), ...folderArray, analyseFiles.shaders[shader].name));
			continue;
		}

		console.group("Validating "+analyseFiles.shaders[shader].name+"...");
		verifyShader(rulesPresets, globalShaders[path.basename(analyseFiles.shaders[shader].name, ".txt")], analyseFiles.shaders[shader].shaderText, analyseFiles.shaders[shader].vulkanSet, path.join(process.cwd(), ...folderArray, analyseFiles.shaders[shader].name));
		console.groupEnd();
	}
}

function verifyGraphicPacks(folderArray) {
	let dirEntries = fs.readdirSync(path.join(process.cwd(), ...folderArray), {withFileTypes: true});
	
	for (let entry in dirEntries) {
		if (dirEntries[entry].isDirectory()) {
			console.group("Verify "+path.join(process.cwd(), ...folderArray, dirEntries[entry].name));

			let packFiles = fs.readdirSync(path.join(process.cwd(), ...folderArray, dirEntries[entry].name), {withFileTypes: true});
			
			let verifyFiles = {rulesText: undefined, shaders: []};
			for (file in packFiles) {
				if (packFiles[file].isFile() && packFiles[file].name == "rules.txt") {
					verifyFiles.rulesText = fs.readFileSync(path.join(process.cwd(), ...folderArray, dirEntries[entry].name, "rules.txt"), {encoding: "utf8"});
				}
				else if (packFiles[file].isFile() && packFiles[file].name.length >= "0000000000000000_0000000000000000_xx.txt".length && /^[a-zA-Z0-9]{16}_[\w]{16}_[p|v]s/.test(packFiles[file].name)) {
					verifyFiles.shaders.push({name: packFiles[file].name, shaderText: fs.readFileSync(path.join(process.cwd(), ...folderArray, dirEntries[entry].name, packFiles[file].name), {encoding: "utf8"}), vulkanSet: ((packFiles[file].name.substr(34, 2) == "vs") ? 0 : 1)});
				}
				else if (packFiles[file].isDirectory()) {
					verifyGraphicPacks([...folderArray, dirEntries[entry].name]);
					break;
				}
			}
			
			verifyPack(verifyFiles, [...folderArray, dirEntries[entry].name]);
			
			console.groupEnd();
		}
	}
}

function extractDumpInfo(folderPath) {
	shaderEntries = fs.readdirSync(folderPath, {withFileTypes: true});
	for (entry in shaderEntries) {
		let currShaderEntry = shaderEntries[entry];
		if (!currShaderEntry.isFile() || path.extname(currShaderEntry.name) != ".txt" || path.basename(currShaderEntry.name, ".txt").slice(-2) == "gs") continue;

		globalShaders[path.basename(currShaderEntry.name, ".txt")] = extractShaderInfo(fs.readFileSync(folderPath+"/"+currShaderEntry.name, {encoding: "utf8"}));
	}
}

extractDumpInfo("./dump/shaders/");

console.info("Finished gathering information from shader dump.");

verifyGraphicPacks(["graphicPacks", "Enhancements"]);
verifyGraphicPacks(["graphicPacks", "Resolutions"]);
verifyGraphicPacks(["graphicPacks", "Workarounds"]);
verifyGraphicPacks(["graphicPacks", "Mods"]);

console.info("Finished verifying the graphic packs!");
console.info("");
console.info("Verified shaders:");
console.info(verifiedShaders.join("\r\n"));
console.info("Invalid shaders:");
console.info(invalidShaders.join("\r\n"));
console.info("Unverified shaders:");
console.info(unverifiedShaders.join("\r\n"));

if (fs.existsSync("./manual/")) fs.rmdirSync("./manual/", {recursive: true});
fs.mkdirSync("./manual/", {recursive: true});

for (file in invalidShaders) {
	fs.mkdirSync("./manual/"+path.basename(invalidShaders[file], ".txt")+"/", {recursive: true});
	fs.copyFileSync("./dump/shaders/"+path.basename(invalidShaders[file]), "./manual/"+path.basename(invalidShaders[file], ".txt")+"/"+path.basename(invalidShaders[file], ".txt")+".dump");
	fs.copyFileSync(invalidShaders[file], "./manual/"+path.basename(invalidShaders[file], ".txt")+"/"+path.basename(invalidShaders[file]));
}