tools: Update implcheck to compare rpl exports with an exports.h file.

Will verify that you do not have any extra exports which should not exist.
Can optionally print unimplemented / implemented function & data exports.

tools/Makefile

@@ -1,6 +1,6 @@
 .SUFFIXES:
 
-TARGETS := elf2rpl readrpl
+TARGETS := elf2rpl readrpl implcheck
 
 ifeq ($(OS),Windows_NT)
 all:

tools/implcheck/Makefile

@@ -0,0 +1,27 @@
+TARGET   := implcheck
+SOURCE   := .
+INCLUDE  := ../common ../ext/cppformat ../ext/excmd/src
+
+CXX      := g++
+
+CPPFORMATSRC := ../ext/cppformat/format.cc
+
+CFILES   := $(foreach dir,$(SOURCE),$(wildcard $(dir)/*.cpp))
+INCLUDES := $(foreach dir,$(INCLUDE),-I$(dir))
+
+CFLAGS   := -O2 -Wall --std=c++14
+LDFLAGS  := -lz
+
+all: $(TARGET)
+
+clean:
+	@echo "[RM] $(notdir $(TARGET))"
+	@rm -rf $(TARGET)
+
+install: all
+	@echo Installing $(TARGET)
+	@cp $(TARGET) $(WUT_ROOT)/bin/$(TARGET)
+
+$(TARGET): $(CFILES)
+	@echo "[CXX] $(notdir $<)"
+	$(CXX) $(CFLAGS) $(INCLUDES) $< -o $@ $(LDFLAGS) $(CPPFORMATSRC)

tools/implcheck/implcheck.vcxproj

@@ -76,6 +76,7 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
     <OutDir>$(SolutionDir)\bin\</OutDir>
+    <IncludePath>$(SolutionDir)\common;$(SolutionDir)\ext\cppformat;$(SolutionDir)\ext\excmd\src;$(SolutionDir)\ext\zlib;$(IncludePath)</IncludePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
@@ -84,6 +85,7 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
     <OutDir>$(SolutionDir)\bin\</OutDir>
+    <IncludePath>$(SolutionDir)\common;$(SolutionDir)\ext\cppformat;$(SolutionDir)\ext\excmd\src;$(SolutionDir)\ext\zlib;$(IncludePath)</IncludePath>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -146,6 +148,14 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="..\ext\zlib\adler32.c" />
+    <ClCompile Include="..\ext\zlib\crc32.c" />
+    <ClCompile Include="..\ext\zlib\infback.c" />
+    <ClCompile Include="..\ext\zlib\inffast.c" />
+    <ClCompile Include="..\ext\zlib\inflate.c" />
+    <ClCompile Include="..\ext\zlib\inftrees.c" />
+    <ClCompile Include="..\ext\zlib\uncompr.c" />
+    <ClCompile Include="..\ext\zlib\zutil.c" />
     <ClCompile Include="main.cpp" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

tools/implcheck/implcheck.vcxproj.filters

@@ -13,10 +13,37 @@
       <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
       <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
     </Filter>
+    <Filter Include="Source Files\ext">
+      <UniqueIdentifier>{d98b1134-aad7-462b-9790-a12b0f037971}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="main.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\ext\zlib\inflate.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
+    <ClCompile Include="..\ext\zlib\adler32.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
+    <ClCompile Include="..\ext\zlib\crc32.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
+    <ClCompile Include="..\ext\zlib\infback.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
+    <ClCompile Include="..\ext\zlib\inffast.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
+    <ClCompile Include="..\ext\zlib\inftrees.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
+    <ClCompile Include="..\ext\zlib\uncompr.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
+    <ClCompile Include="..\ext\zlib\zutil.c">
+      <Filter>Source Files\ext</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

tools/implcheck/main.cpp

@@ -1,100 +1,339 @@
+#include <algorithm>
+#include <excmd.h>
+#include <format.h>
 #include <fstream>
-#include <set>
-#include <string>
 #include <iostream>
+#include <vector>
+#include <set>
+#include <zlib.h>
+#include "elf.h"
 
-int main(int argc, char **argv)
+struct Section
 {
-   std::set<std::string> funcExports, dataExports;
+   elf::SectionHeader header;
-   std::string line;
+   std::vector<char> data;
-   std::ifstream in;
+};
-   std::ofstream out;
 
-   if (argc < 4) {
+struct RplInfo
-      std::cout << argv[0] << " <readrpl output> <exports.h> <output.txt>" << std::endl;
+{
-      return 0;
+   std::set<std::string> funcExports;
+   std::set<std::string> dataExports;
+};
+
+struct ExportsInfo
+{
+   std::set<std::string> funcExports;
+   std::set<std::string> dataExports;
+};
+
+static bool
+readSection(std::ifstream &fh,
+            elf::SectionHeader &header,
+            std::vector<char> &data)
+{
+   // Read section header
+   fh.read(reinterpret_cast<char*>(&header), sizeof(elf::SectionHeader));
+
+   if (header.type == elf::SHT_NOBITS || !header.size) {
+      return true;
    }
 
-   in.open(argv[1]);
+   // Read section data
+   if (header.flags & elf::SHF_DEFLATED) {
+      auto stream = z_stream {};
+      auto ret = Z_OK;
 
-   if (!in.is_open()) {
+      // Read the original size
-      std::cout << "Could not open file " << argv[1] << " for reading" << std::endl;
+      fh.seekg(header.offset.value());
-      return -1;
+      be_val<uint32_t> size = 0;
-   }
+      fh.read(reinterpret_cast<char *>(&size), sizeof(uint32_t));
+      data.resize(size);
 
+      // Inflate
+      memset(&stream, 0, sizeof(stream));
+      stream.zalloc = Z_NULL;
+      stream.zfree = Z_NULL;
+      stream.opaque = Z_NULL;
 
-   while (!in.eof()) {
+      ret = inflateInit(&stream);
-      std::string strAddr, name;
-      in >> strAddr >> name;
 
-      auto addr = std::stoul(strAddr, nullptr, 16);
+      if (ret != Z_OK) {
-
+         std::cout << "Couldn't decompress .rpx section because inflateInit returned " << ret << std::endl;
-      if (addr >= 0x10000000) {
+         data.clear();
-         dataExports.insert(name);
+         return false;
       } else {
-         funcExports.insert(name);
+         std::vector<char> temp;
+         temp.resize(header.size - sizeof(uint32_t));
+         fh.read(temp.data(), temp.size());
+
+         stream.avail_in = header.size;
+         stream.next_in = reinterpret_cast<Bytef*>(temp.data());
+         stream.avail_out = static_cast<uInt>(data.size());
+         stream.next_out = reinterpret_cast<Bytef*>(data.data());
+
+         ret = inflate(&stream, Z_FINISH);
+
+         if (ret != Z_OK && ret != Z_STREAM_END) {
+            std::cout << "Couldn't decompress .rpx section because inflate returned " << ret << std::endl;
+            data.clear();
+            return false;
+         }
+
+         inflateEnd(&stream);
+      }
+   } else {
+      data.resize(header.size);
+      fh.seekg(header.offset.value());
+      fh.read(data.data(), header.size);
+   }
+
+   return true;
+}
+
+static void
+readExports(RplInfo &info, const Section &section)
+{
+   auto exports = reinterpret_cast<const elf::RplExport *>(section.data.data());
+   auto strTab = section.data.data();
+
+   for (auto i = 0u; i < exports->count; ++i) {
+      // TLS exports have the high bit set in name for some unknown reason...
+      auto name = strTab + (exports->exports[i].name & 0x7FFFFFFF);
+      auto value = exports->exports[i].value;
+
+      if (value >= 0x10000000) {
+         info.dataExports.insert(name);
+      } else {
+         info.funcExports.insert(name);
+      }
+   }
+}
+
+static bool
+readRPL(RplInfo &info, std::ifstream &fh)
+{
+   elf::Header header;
+   fh.read(reinterpret_cast<char*>(&header), sizeof(elf::Header));
+
+   if (header.magic != elf::HeaderMagic) {
+      std::cout << "Invalid ELF magic header" << std::endl;
+      return false;
+   }
+
+   // Read sections
+   auto sections = std::vector<Section> {};
+
+   for (auto i = 0u; i < header.shnum; ++i) {
+      Section section;
+      fh.seekg(header.shoff + header.shentsize * i);
+
+      if (!readSection(fh, section.header, section.data)) {
+         std::cout << "Error reading section " << i << std::endl;
+         return false;
+      }
+
+      sections.push_back(section);
+   }
+
+   // Find strtab
+   auto shStrTab = reinterpret_cast<const char *>(sections[header.shstrndx].data.data());
+
+   // Print section data
+   for (auto i = 0u; i < sections.size(); ++i) {
+      auto &section = sections[i];
+
+      if (section.header.type == elf::SHT_RPL_EXPORTS) {
+         readExports(info, section);
       }
    }
 
-   in.close();
+   return true;
-   in.open(argv[2]);
+}
 
-   if (!in.is_open()) {
+static bool
-      std::cout << "Could not open file " << argv[2] << " for reading" << std::endl;
+readTextExports(ExportsInfo &info, std::ifstream &fh)
-      return -1;
+{
-   }
+   std::string line;
 
-   auto funcCount = funcExports.size();
+   while (std::getline(fh, line)) {
-   auto dataCount = funcExports.size();
-
-   while (std::getline(in, line)) {
       if (line.find("EXPORT(") == 0) {
          auto name = line.substr(strlen("EXPORT("));
          name = name.erase(name.find(')'), 2);
-
+         info.funcExports.insert(name);
-         auto funcItr = funcExports.find(name);
-
-         if (funcItr != funcExports.end()) {
-            funcExports.erase(funcItr);
-         }
-
-         auto dataItr = dataExports.find(name);
-
-         if (dataItr != dataExports.end()) {
-            dataExports.erase(funcItr);
-         }
       }
    }
 
-   auto unimplFuncCount = funcExports.size();
+   return true;
-   auto unimplDataCount = funcExports.size();
+}
 
-   out.open(argv[3]);
+int main(int argc, char **argv)
+{
+   excmd::parser parser;
+   excmd::option_state options;
+   using excmd::description;
+   using excmd::value;
 
-   if (!out.is_open()) {
+   try {
-      std::cout << "Could not open file " << argv[3] << " for writing" << std::endl;
+      parser.global_options()
+         .add_option("H,help",
+                     description { "Show help." })
+         .add_option("f,functions",
+                     description { "Print unimplemented function exports." })
+         .add_option("d,data",
+                     description { "Print unimplemented data exports." });
+
+      parser.default_command()
+         .add_argument("rpl-path",
+                       description { "Path to RPL file" },
+                       value<std::string> {})
+         .add_argument("exports-path",
+                       description { "Path to exports.h file" },
+                       value<std::string> {});
+
+      options = parser.parse(argc, argv);
+   } catch (excmd::exception ex) {
+      std::cout << "Error parsing options: " << ex.what() << std::endl;
       return -1;
    }
 
-   if (funcExports.size()) {
+   if (options.empty() || options.has("help") || !options.has("rpl-path") || !options.has("exports-path")) {
-      out << "Unimplemented function exports " << unimplFuncCount << "/" << funcCount << ":" << std::endl;
+      std::cout << argv[0] << " rpl-path exports-path" << std::endl;
-
+      std::cout << parser.format_help(argv[0]);
-      for (auto &name : funcExports) {
+      return 0;
-         out << name << std::endl;
-      }
-
-      out << std::endl;
    }
 
-   if (dataExports.size()) {
+   auto rplPath = options.get<std::string>("rpl-path");
-      out << "Unimplemented data exports " << unimplDataCount << "/" << dataCount << ":" << std::endl;
+   auto exportsPath = options.get<std::string>("exports-path");
 
-      for (auto &name : dataExports) {
+   // Read RPL
-         out << name << std::endl;
+   std::ifstream fh { rplPath, std::ifstream::binary };
-      }
 
-      out << std::endl;
+   if (!fh.is_open()) {
+      std::cout << "Could not open " << rplPath << " for reading" << std::endl;
+      return -1;
    }
 
-   return 0;
+   RplInfo rplInfo;
+
+   if (!readRPL(rplInfo, fh)) {
+      std::cout << "Error reading " << rplPath << std::endl;
+      return -1;
+   }
+
+   fh.close();
+
+   // Read exports
+   fh.open(exportsPath, std::ifstream::in);
+
+   if (!fh.is_open()) {
+      std::cout << "Could not open " << exportsPath << " for reading" << std::endl;
+      return -1;
+   }
+
+   ExportsInfo exportsInfo;
+
+   if (!readTextExports(exportsInfo, fh)) {
+      std::cout << "Error reading " << exportsPath << std::endl;
+      return -1;
+   }
+
+   fh.close();
+
+   // Check that we do not have any extra exports which are not exported from RPL.
+   std::set<std::string> funcUnion, extraExports;
+   int result = 0;
+
+   std::set_union(rplInfo.funcExports.begin(), rplInfo.funcExports.end(),
+                  exportsInfo.funcExports.begin(), exportsInfo.funcExports.end(),
+                  std::inserter(funcUnion, funcUnion.begin()));
+
+   std::set_difference(funcUnion.begin(), funcUnion.end(),
+                       rplInfo.funcExports.begin(), rplInfo.funcExports.end(),
+                       std::inserter(extraExports, extraExports.begin()));
+
+   if (!extraExports.empty()) {
+      std::cout << "Error: Exports found in exports.h which do not exist in rpl:" << std::endl;
+
+      for (auto &name : extraExports) {
+         std::cout << " - " << name << std::endl;
+      }
+
+      std::cout << std::endl;
+      result = -1;
+   }
+
+   if (options.has("functions")) {
+      // Print implemented function exports
+      std::set<std::string> intersection;
+
+      std::set_intersection(rplInfo.funcExports.begin(), rplInfo.funcExports.end(),
+                            exportsInfo.funcExports.begin(), exportsInfo.funcExports.end(),
+                            std::inserter(intersection, intersection.begin()));
+
+      if (!intersection.empty()) {
+         std::cout << "Implemented function exports:" << std::endl;
+
+         for (auto &name : intersection) {
+            std::cout << " - " << name << std::endl;
+         }
+
+         std::cout << std::endl;
+      }
+
+      // Print unimplemented function exports
+      std::set<std::string> difference;
+
+      std::set_difference(rplInfo.funcExports.begin(), rplInfo.funcExports.end(),
+                          exportsInfo.funcExports.begin(), exportsInfo.funcExports.end(),
+                          std::inserter(difference, difference.begin()));
+
+      if (!difference.empty()) {
+         std::cout << "Unimplemented function exports:" << std::endl;
+
+         for (auto &name : difference) {
+            std::cout << " - " << name << std::endl;
+         }
+
+         std::cout << std::endl;
+      }
+   }
+
+   if (options.has("data")) {
+      // Print implemented data exports
+      std::set<std::string> intersection;
+
+      std::set_intersection(rplInfo.dataExports.begin(), rplInfo.dataExports.end(),
+                            exportsInfo.dataExports.begin(), exportsInfo.dataExports.end(),
+                            std::inserter(intersection, intersection.begin()));
+
+      if (!intersection.empty()) {
+         std::cout << "Implemented data exports:" << std::endl;
+
+         for (auto &name : intersection) {
+            std::cout << " - " << name << std::endl;
+         }
+
+         std::cout << std::endl;
+      }
+
+      // Print unimplemented data exports
+      std::set<std::string> difference;
+
+      std::set_difference(rplInfo.dataExports.begin(), rplInfo.dataExports.end(),
+                          exportsInfo.dataExports.begin(), exportsInfo.dataExports.end(),
+                          std::inserter(difference, difference.begin()));
+
+      if (!difference.empty()) {
+         std::cout << "Unimplemented data exports:" << std::endl;
+
+         for (auto &name : difference) {
+            std::cout << " - " << name << std::endl;
+         }
+
+         std::cout << std::endl;
+      }
+   }
+
+   return result;
 }
+