From 755f7c75afc8834ec8dca64d189b5c496d502d08 Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Sat, 10 Dec 2022 15:32:23 +0000 Subject: [PATCH] Add pipeline (de)serialisation support to bundle See comments in code for details on the on-disk format. --- .../common/pipeline_state_bundle.cpp | 149 +++++++++++++++++- .../common/pipeline_state_bundle.h | 25 ++- 2 files changed, 164 insertions(+), 10 deletions(-) diff --git a/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.cpp b/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.cpp index 97954629..a10e694d 100644 --- a/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.cpp +++ b/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.cpp @@ -30,7 +30,7 @@ namespace skyline::gpu::interconnect { } void PipelineStateBundle::AddTextureType(u32 index, Shader::TextureType type) { - textureTypes.emplace_back(index, type); + textureTypes.push_back({index, type}); } void PipelineStateBundle::AddConstantBufferValue(u32 shaderStage, u32 index, u32 offset, u32 value) { @@ -46,19 +46,156 @@ namespace skyline::gpu::interconnect { return {stageInfo.binary, stageInfo.binaryBaseOffset}; } - Shader::TextureType PipelineStateBundle::LookupTextureType(u32 offset) { - auto it{ranges::find_if(textureTypes, [offset](auto &pair) { return pair.first == offset; })}; + Shader::TextureType PipelineStateBundle::LookupTextureType(u32 index) { + auto it{ranges::find_if(textureTypes, [index](const auto &entry) { return entry.index == index; })}; if (it == textureTypes.end()) - throw exception("Failed to find texture type for offset: 0x{:X}", offset); + throw exception("Failed to find texture type for index: 0x{:X}", index); - return it->second; + return it->type; } u32 PipelineStateBundle::LookupConstantBufferValue(u32 shaderStage, u32 index, u32 offset) { - auto it{ranges::find_if(constantBufferValues, [index, offset, shaderStage](auto &val) { return val.index == index && val.offset == offset && val.shaderStage == shaderStage; })}; + auto it{ranges::find_if(constantBufferValues, [index, offset, shaderStage](const auto &val) { return val.index == index && val.offset == offset && val.shaderStage == shaderStage; })}; if (it == constantBufferValues.end()) throw exception("Failed to find constant buffer value for offset: 0x{:X}", offset); return it->value; } + + static constexpr u32 MaxSerialisedBundleSize{1 << 20}; ///< The maximum size of a serialised bundle (1 MiB) + + /* Bundle header format pseudocode: + u64 hash + u32 bundleSize + u32 keySize; + u32 constantBufferValueCount + u32 textureTypeCount + u32 pipelineStageCount + u8 key[keySize]; + + struct ConstantBufferValue { + u32 shaderStage; + u32 index; + u32 offset; + u32 value; + } constantBufferValues[constantBufferValueCount]; + + struct TextureType { + u32 index; + u32 (Shader::TextureType) type; + } textureType[textureTypeCount]; + + struct PipelineStage { + u32 binaryBaseOffset + u32 binarySize + u8 binary[binarySize] + } pipelineStages[pipelineStageCount]; + */ + + struct BundleDataHeader { + u32 keySize; + u32 constantBufferValueCount; + u32 textureTypeCount; + u32 pipelineStageCount; + }; + + struct PipelineBinaryDataHeader { + u32 binaryBaseOffset; + u32 binarySize; + }; + + bool PipelineStateBundle::Deserialise(std::ifstream &stream) { + if (stream.peek() == EOF) + return false; + + u64 hash{}; + stream.read(reinterpret_cast(&hash), sizeof(hash)); + + u32 bundleSize{}; + stream.read(reinterpret_cast(&bundleSize), sizeof(bundleSize)); + if (bundleSize > MaxSerialisedBundleSize) + throw exception("Pipeline state bundle is too large: 0x{:X}", bundleSize); + + fileBuffer.resize(static_cast(bundleSize)); + stream.read(reinterpret_cast(fileBuffer.data()), static_cast(bundleSize)); + + if (XXH64(fileBuffer.data(), bundleSize, 0) != hash) + throw exception("Pipeline state bundle hash mismatch"); + + auto data{span(fileBuffer)}; + const auto &header{data.as()}; + size_t offset{sizeof(BundleDataHeader)}; + + Reset(data.subspan(offset, header.keySize)); + offset += header.keySize; + + auto readConstantBufferValues{data.subspan(offset, header.constantBufferValueCount * sizeof(ConstantBufferValue)).cast()}; + textureTypes.reserve(header.textureTypeCount); + constantBufferValues.insert(constantBufferValues.end(), readConstantBufferValues.begin(), readConstantBufferValues.end()); + offset += header.constantBufferValueCount * sizeof(ConstantBufferValue); + + auto readTextureTypes{data.subspan(offset, header.textureTypeCount * sizeof(TextureTypeEntry)).cast()}; + textureTypes.reserve(header.textureTypeCount); + textureTypes.insert(textureTypes.end(), readTextureTypes.begin(), readTextureTypes.end()); + offset += header.textureTypeCount * sizeof(TextureTypeEntry); + + pipelineStages.resize(header.pipelineStageCount); + for (u32 i{}; i < header.pipelineStageCount; i++) { + const auto &pipelineHeader{data.subspan(offset).as()}; + offset += sizeof(PipelineBinaryDataHeader); + + pipelineStages[i].binaryBaseOffset = pipelineHeader.binaryBaseOffset; + pipelineStages[i].binary.resize(pipelineHeader.binarySize); + span(pipelineStages[i].binary).copy_from(data.subspan(offset, pipelineHeader.binarySize)); + offset += pipelineHeader.binarySize; + } + + return true; + } + + void PipelineStateBundle::Serialise(std::ofstream &stream) { + u32 bundleSize{static_cast(sizeof(BundleDataHeader) + + key.size() + + constantBufferValues.size() * sizeof(ConstantBufferValue) + + textureTypes.size() * sizeof(TextureTypeEntry) + + std::accumulate(pipelineStages.begin(), pipelineStages.end(), 0UL, [](size_t acc, const auto &stage) { + return acc + sizeof(PipelineBinaryDataHeader) + stage.binary.size(); + }))}; + + fileBuffer.resize(bundleSize); + + auto data{span(fileBuffer)}; + auto &header{data.as()}; + size_t offset{sizeof(BundleDataHeader)}; + + header.keySize = static_cast(key.size()); + header.constantBufferValueCount = static_cast(constantBufferValues.size()); + header.textureTypeCount = static_cast(textureTypes.size()); + header.pipelineStageCount = static_cast(pipelineStages.size()); + + data.subspan(offset, header.keySize).copy_from(key); + offset += header.keySize; + + data.subspan(offset, header.constantBufferValueCount * sizeof(ConstantBufferValue)).copy_from(constantBufferValues); + offset += header.constantBufferValueCount * sizeof(ConstantBufferValue); + + data.subspan(offset, header.textureTypeCount * sizeof(TextureTypeEntry)).copy_from(textureTypes); + offset += header.textureTypeCount * sizeof(TextureTypeEntry); + + for (const auto &stage : pipelineStages) { + auto &pipelineHeader{data.subspan(offset).as()}; + offset += sizeof(PipelineBinaryDataHeader); + + pipelineHeader.binaryBaseOffset = stage.binaryBaseOffset; + pipelineHeader.binarySize = static_cast(stage.binary.size()); + + data.subspan(offset, pipelineHeader.binarySize).copy_from(stage.binary); + offset += pipelineHeader.binarySize; + } + + u64 hash{XXH64(fileBuffer.data(), bundleSize, 0)}; + stream.write(reinterpret_cast(&hash), sizeof(hash)); + stream.write(reinterpret_cast(&bundleSize), sizeof(bundleSize)); + stream.write(reinterpret_cast(fileBuffer.data()), static_cast(bundleSize)); + } } diff --git a/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.h b/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.h index fec50f4d..bd25ec33 100644 --- a/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.h +++ b/app/src/main/cpp/skyline/gpu/interconnect/common/pipeline_state_bundle.h @@ -13,6 +13,7 @@ namespace skyline::gpu::interconnect { class PipelineStateBundle { private: std::vector key; //!< Byte array containing the pipeline key, this is interpreted by the the user and two different keys might refer to the same pipeline + std::vector fileBuffer; /** * @brief Holds the raw binary and associated info for a pipeline stage @@ -26,6 +27,7 @@ namespace skyline::gpu::interconnect { /** * @brief Holds a value of a constant buffer read from memory at pipeline creation time + * @note This struct *MUST* not be modified without a pipeline cache version bump */ struct ConstantBufferValue { u32 shaderStage; @@ -33,9 +35,20 @@ namespace skyline::gpu::interconnect { u32 offset; u32 value; }; + static_assert(sizeof(ConstantBufferValue) == 0x10); + + /** + * @brief Holds a the texture type of a TIC entry read at pipeline creation time + * @note This struct *MUST* not be modified without a pipeline cache version bump + */ + struct TextureTypeEntry { + u32 index; + Shader::TextureType type; + }; + static_assert(sizeof(TextureTypeEntry) == 0x8); boost::container::small_vector constantBufferValues; - boost::container::small_vector, 4> textureTypes; + boost::container::small_vector textureTypes; std::vector pipelineStages{}; @@ -47,7 +60,7 @@ namespace skyline::gpu::interconnect { */ void Reset(span newKey); - template requires std::is_trivially_copyable_v + template requires std::is_trivially_copyable_v && (!requires (T t){ t.size(); }) void Reset(const T &value) { Reset(span(reinterpret_cast(&value), sizeof(T))); } @@ -74,7 +87,7 @@ namespace skyline::gpu::interconnect { template requires std::is_trivially_copyable_v const T &GetKey() { - return *reinterpret_cast(key.data()); + return GetKey().as(); } /** @@ -85,11 +98,15 @@ namespace skyline::gpu::interconnect { /** * @brief Returns the texture type for a given offset */ - Shader::TextureType LookupTextureType(u32 offset); + Shader::TextureType LookupTextureType(u32 index); /** * @brief Returns the constant buffer value for a given offset and shader stage */ u32 LookupConstantBufferValue(u32 shaderStage, u32 index, u32 offset); + + bool Deserialise(std::ifstream &stream); + + void Serialise(std::ofstream &stream); }; }