diff --git a/Source/Core/VideoCommon/NativeVertexFormat.h b/Source/Core/VideoCommon/NativeVertexFormat.h
index ebc95cdc45..8778423d3f 100644
--- a/Source/Core/VideoCommon/NativeVertexFormat.h
+++ b/Source/Core/VideoCommon/NativeVertexFormat.h
@@ -4,7 +4,7 @@
 
 #pragma once
 
-#include "Common/CommonTypes.h"
+#include "Common/Hash.h"
 
 // m_components
 enum
@@ -87,6 +87,20 @@ struct PortableVertexDeclaration
 	}
 };
 
+namespace std
+{
+
+template <>
+struct hash<PortableVertexDeclaration>
+{
+	size_t operator()(const PortableVertexDeclaration& decl) const
+	{
+		return HashFletcher((u8 *) &decl, sizeof(decl));
+	}
+};
+
+}
+
 // The implementation of this class is specific for GL/DX, so NativeVertexFormat.cpp
 // is in the respective backend, not here in VideoCommon.
 
diff --git a/Source/Core/VideoCommon/VertexLoader.cpp b/Source/Core/VideoCommon/VertexLoader.cpp
index ebc57d8577..ab78e85f9e 100644
--- a/Source/Core/VideoCommon/VertexLoader.cpp
+++ b/Source/Core/VideoCommon/VertexLoader.cpp
@@ -1051,4 +1051,4 @@ NativeVertexFormat* VertexLoader::GetNativeVertexFormat()
 
 }
 
-std::map<PortableVertexDeclaration, std::unique_ptr<NativeVertexFormat>> VertexLoader::s_native_vertex_map;
+std::unordered_map<PortableVertexDeclaration, std::unique_ptr<NativeVertexFormat>> VertexLoader::s_native_vertex_map;
diff --git a/Source/Core/VideoCommon/VertexLoader.h b/Source/Core/VideoCommon/VertexLoader.h
index 0bca6f5e5c..c81be52556 100644
--- a/Source/Core/VideoCommon/VertexLoader.h
+++ b/Source/Core/VideoCommon/VertexLoader.h
@@ -8,9 +8,9 @@
 // Metroid Prime: P I16-flt N I16-s16 T0 I16-u16 T1 i16-flt
 
 #include <algorithm>
-#include <map>
 #include <memory>
 #include <string>
+#include <unordered_map>
 
 #include "Common/CommonTypes.h"
 #include "Common/x64Emitter.h"
@@ -141,7 +141,7 @@ private:
 	int m_numLoadedVertices;
 
 	NativeVertexFormat* m_native_vertex_format;
-	static std::map<PortableVertexDeclaration, std::unique_ptr<NativeVertexFormat>> s_native_vertex_map;
+	static std::unordered_map<PortableVertexDeclaration, std::unique_ptr<NativeVertexFormat>> s_native_vertex_map;
 
 	void SetVAT(const VAT& vat);