add external file hashes to the binary paragraph

scripts/get_triplet_environment.cmake

@@ -13,4 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}")
 message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}")
 message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}")
 message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}")
-message("VCPKG_ABI_ADDITIONAL_FILES=${VCPKG_ABI_ADDITIONAL_FILES}")
+message("VCPKG_EXTERNAL_FILES=${VCPKG_EXTERNAL_FILES}")

toolsrc/include/vcpkg/binaryparagraph.h

@@ -32,6 +32,7 @@ namespace vcpkg
         std::vector<std::string> depends;
         std::string abi;
         SourceParagraph::TYPE type;
+        std::unordered_map<std::string, std::string> external_files;
     };
 
     struct BinaryControlFile

toolsrc/include/vcpkg/build.h

@@ -138,7 +138,7 @@ namespace vcpkg::Build
         Optional<std::string> external_toolchain_file;
         Optional<ConfigurationType> build_type;
         std::vector<std::string> passthrough_env_vars;
-        std::vector<fs::path> additional_files;
+        std::vector<std::pair<std::string, std::string>> external_files;
     };
 
     std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset);
@@ -153,7 +153,7 @@ namespace vcpkg::Build
         CHAINLOAD_TOOLCHAIN_FILE,
         BUILD_TYPE,
         ENV_PASSTHROUGH,
-        ABI_ADDITIONAL_FILES,
+        EXTERNAL_FILES,
     };
 
     const std::unordered_map<std::string, VcpkgTripletVar> VCPKG_OPTIONS = {
@@ -165,7 +165,7 @@ namespace vcpkg::Build
         {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE},
         {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE},
         {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH},
-        {"VCPKG_ABI_ADDITIONAL_FILES", VcpkgTripletVar::ABI_ADDITIONAL_FILES},
+        {"VCPKG_EXTERNAL_FILES", VcpkgTripletVar::EXTERNAL_FILES},
     };
 
     struct ExtendedBuildResult

toolsrc/src/vcpkg/binaryparagraph.cpp

@@ -2,6 +2,7 @@
 
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/system.print.h>
+#include <vcpkg/base/util.h>
 #include <vcpkg/binaryparagraph.h>
 #include <vcpkg/parse.h>
 
@@ -23,7 +24,7 @@ namespace vcpkg
         static const std::string MAINTAINER = "Maintainer";
         static const std::string DEPENDS = "Depends";
         static const std::string DEFAULTFEATURES = "Default-Features";
-        static const std::string TYPE = "Type";
+        static const std::string EXTERNALFILES = "External-Files";
     }
 
     BinaryParagraph::BinaryParagraph() = default;
@@ -61,8 +62,23 @@ namespace vcpkg
             this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES));
         }
 
-        this->type =
+        std::vector<std::string> external_files_or_hashes =
-            SourceParagraph::type_from_string(parser.optional_field(Fields::TYPE));
+            parse_comma_list(parser.optional_field(Fields::EXTERNALFILES));
+
+        if (external_files_or_hashes.size() % 2 != 0)
+        {
+            Checks::exit_with_message(
+                    VCPKG_LINE_INFO,
+                    "The External-Files field is not composed of key-value pairs for ",
+                    this->spec);
+        }
+
+        for (int i = 0; i < external_files_or_hashes.size(); i += 2)
+        {
+            external_files.emplace(
+                    std::move(external_files_or_hashes[i]),
+                    std::move(external_files_or_hashes[i+1]));
+        }
 
         if (const auto err = parser.error_info(this->spec.to_string()))
         {
@@ -124,6 +140,16 @@ namespace vcpkg
         if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n');
         if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n');
 
-        out_str.append("Type: ").append(SourceParagraph::string_from_type(pgh.type)).push_back('\n');
+        if (!pgh.external_files.empty())
+        {
+            out_str.append("External-Files: ");
+            out_str.append(Strings::join(",",
+                        Util::fmap(
+                            pgh.external_files,
+                            [](const std::pair<std::string, std::string>& kv)
+                            {
+                                return kv.first + "," + kv.second;
+                            }))).push_back('\n');
+        }
     }
 }

toolsrc/src/vcpkg/build.cpp

@@ -9,6 +9,7 @@
 #include <vcpkg/base/system.debug.h>
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/system.process.h>
+#include <vcpkg/base/util.h>
 
 #include <vcpkg/build.h>
 #include <vcpkg/commands.h>
@@ -261,9 +262,11 @@ namespace vcpkg::Build
         return BinaryParagraph(source_paragraph, feature_paragraph, triplet);
     }
 
-    static std::unique_ptr<BinaryControlFile> create_binary_control_file(const SourceParagraph& source_paragraph,
+    static std::unique_ptr<BinaryControlFile> create_binary_control_file(
+        const SourceParagraph& source_paragraph,
         const Triplet& triplet,
         const BuildInfo& build_info,
+        const PreBuildInfo& pre_build_info,
         const std::string& abi_tag)
     {
         auto bcf = std::make_unique<BinaryControlFile>();
@@ -272,6 +275,14 @@ namespace vcpkg::Build
         {
             bpgh.version = *p_ver;
         }
+
+        for (auto& file_hash : pre_build_info.external_files)
+        {
+            bpgh.external_files.emplace(
+                std::move(file_hash.first),
+                std::move(file_hash.second));
+        }
+
         bcf->core_paragraph = std::move(bpgh);
         return bcf;
     }
@@ -447,6 +458,41 @@ namespace vcpkg::Build
         return command;
     }
 
+    static std::vector<std::pair<std::string, std::string>> get_external_file_hashes(
+        const VcpkgPaths& paths,
+        const std::vector<fs::path>& files)
+    {
+        static std::map<fs::path, std::string> s_hash_cache;
+
+        const auto& fs = paths.get_filesystem();
+
+        std::vector<std::pair<std::string, std::string>> hashes;
+        for (const fs::path& external_file : files)
+        {
+            auto it_hash = s_hash_cache.find(external_file);
+
+            if (it_hash != s_hash_cache.end())
+            {
+                hashes.emplace_back(external_file, it_hash->second);
+            }
+            else if (fs.is_regular_file(external_file))
+            {
+                auto emp = s_hash_cache.emplace(external_file.u8string(),
+                                                Hash::get_file_hash(fs, external_file, "SHA1"));
+                hashes.emplace_back(external_file, emp.first->second);
+            }
+            else
+            {
+                Checks::exit_with_message(
+                        VCPKG_LINE_INFO,
+                        external_file.u8string() +
+                        " was listed as an additional file for calculating the abi, but was not found.");
+            }
+        }
+
+        return hashes;
+    }
+
     static std::string get_triplet_abi(const VcpkgPaths& paths,
                                        const PreBuildInfo& pre_build_info,
                                        const Triplet& triplet)
@@ -496,31 +542,6 @@ namespace vcpkg::Build
             s_hash_cache.emplace(triplet_file_path, hash);
         }
 
-        for (const fs::path& additional_file : pre_build_info.additional_files)
-        {
-            it_hash = s_hash_cache.find(additional_file);
-
-            if (it_hash != s_hash_cache.end())
-            {
-                hash += "-";
-                hash += it_hash->second;
-            }
-            else if (fs.is_regular_file(additional_file))
-            {
-                std::string tmp_hash = Hash::get_file_hash(fs, additional_file, "SHA1");
-                hash += "-";
-                hash += tmp_hash;
-
-                s_hash_cache.emplace(additional_file, tmp_hash);
-            }
-            else
-            {
-                Checks::exit_with_message(
-                        VCPKG_LINE_INFO,
-                        additional_file.u8string() + " was listed as an additional file for calculating the abi, but was not found.");
-            }
-        }
-
         return hash;
     }
 
@@ -572,7 +593,13 @@ namespace vcpkg::Build
         const size_t error_count =
             PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir);
 
-        auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag);
+        std::unique_ptr<BinaryControlFile> bcf =
+            create_binary_control_file(
+                *config.scf.core_paragraph,
+                triplet,
+                build_info,
+                pre_build_info,
+                abi_tag);
 
         if (error_count != 0)
         {
@@ -629,7 +656,9 @@ namespace vcpkg::Build
 
         std::vector<AbiEntry> abi_tag_entries(dependency_abis.begin(), dependency_abis.end());
 
-        abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)});
+        // Sorted here as the order of dependency_abis is the only
+        // non-deterministicly ordered set of AbiEntries
+        Util::sort(abi_tag_entries);
 
         // If there is an unusually large number of files in the port then
         // something suspicious is going on.  Rather than hash all of them
@@ -637,13 +666,16 @@ namespace vcpkg::Build
         const int max_port_file_count = 100;
 
         // the order of recursive_directory_iterator is undefined so save the names to sort
-        std::vector<fs::path> port_files;
+        std::vector<std::pair<std::string, std::string>> hashes_files;
         for (auto& port_file : fs::stdfs::recursive_directory_iterator(config.port_dir))
         {
             if (fs::is_regular_file(status(port_file)))
             {
-                port_files.push_back(port_file);
+                hashes_files.emplace_back(
-                if (port_files.size() > max_port_file_count)
+                    vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"),
+                    port_file.path().filename());
+
+                if (hashes_files.size() > max_port_file_count)
                 {
                     abi_tag_entries.emplace_back(AbiEntry{"no_hash_max_portfile", ""});
                     break;
@@ -651,25 +683,45 @@ namespace vcpkg::Build
             }
         }
 
-        if (port_files.size() <= max_port_file_count)
+        if (hashes_files.size() <= max_port_file_count)
         {
-            std::sort(port_files.begin(), port_files.end());
+            Util::sort(hashes_files);
 
-            int counter = 0;
+            for (auto& hash_file : hashes_files)
-            for (auto& port_file : port_files)
             {
-                // When vcpkg takes a dependency on C++17 it can use fs::relative,
+                // We've already sorted by hash so it's safe to write down the
-                // which will give a stable ordering and better names in the key entry.
+                // filename, which will be consistent across machines.
-                // this is not available in the filesystem TS so instead number the files for the key.
+                abi_tag_entries.emplace_back(
-                std::string key = Strings::format("file_%03d", counter++);
+                    AbiEntry{
-                if (Debug::g_debugging)
+                        std::move(hash_file.second),
+                        std::move(hash_file.first)
+                    });
+            }
+        }
+
+        //Make a copy of the external files and their hashes, and sort by hash
+        auto additional_file_hashes = pre_build_info.external_files;
+        std::sort(
+            additional_file_hashes.begin(),
+            additional_file_hashes.end(),
+            [](const std::pair<fs::path, std::string>& l,
+               const std::pair<fs::path, std::string>& r)
             {
-                    System::print2("[DEBUG] mapping ", key, " from ", port_file.u8string(), "\n");
+                return l.second < r.second ||
-                }
+                    (l.second == r.second && l.first < r.first);
-                abi_tag_entries.emplace_back(AbiEntry{key, vcpkg::Hash::get_file_hash(fs, port_file, "SHA1")});
+            });
-            }
+
+        for (auto& hash_file : additional_file_hashes)
+        {
+            abi_tag_entries.emplace_back(
+                AbiEntry{
+                    std::move(hash_file.first),
+                    std::move(hash_file.second)
+                });
         }
 
+        abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)});
+
         abi_tag_entries.emplace_back(AbiEntry{
             "vcpkg_fixup_cmake_targets",
             vcpkg::Hash::get_file_hash(fs, paths.scripts / "cmake" / "vcpkg_fixup_cmake_targets.cmake", "SHA1")});
@@ -682,8 +734,6 @@ namespace vcpkg::Build
         if (config.build_package_options.use_head_version == UseHeadVersion::YES)
             abi_tag_entries.emplace_back(AbiEntry{"head", ""});
 
-        Util::sort(abi_tag_entries);
-
         const std::string full_abi_info =
             Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; });
 
@@ -1114,12 +1164,15 @@ namespace vcpkg::Build
                 case VcpkgTripletVar::ENV_PASSTHROUGH :
                     pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";");
                     break;
-                case VcpkgTripletVar::ABI_ADDITIONAL_FILES :
+                case VcpkgTripletVar::EXTERNAL_FILES :
-                    pre_build_info.additional_files = Util::fmap(Strings::split(variable_value, ";"),
+                    pre_build_info.external_files = 
+                        get_external_file_hashes(
+                            paths,
+                            Util::fmap(Strings::split(variable_value, ";"),
                             [](const std::string& path)
                             {
                                 return fs::path{path};
-                            });
+                            }));
                     break;
                 }
             }

toolsrc/src/vcpkg/statusparagraphs.cpp

@@ -118,30 +118,30 @@ namespace vcpkg
         return it != end() && (*it)->is_installed();
     }
 
-    bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec)
+    //bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec)
-    {
+    //{
-        auto it = find(spec);
+    //    auto it = find(spec);
-        if (it != end())
+    //    if (it != end())
-        {
+    //    {
-            for (const std::string& dep : (*it)->package.depends)
+    //        for (const std::string& dep : (*it)->package.depends)
-            {
+    //        {
-                PackageSpec dep_spec =
+    //            PackageSpec dep_spec =
-                    PackageSpec::from_name_and_triplet(
+    //                PackageSpec::from_name_and_triplet(
-                            dep,
+    //                        dep,
-                            spec.triplet()).value_or_exit(VCPKG_LINE_INFO);
+    //                        spec.triplet()).value_or_exit(VCPKG_LINE_INFO);
 
-                if (needs_rebuild(dep_spec))
+    //            if (needs_rebuild(dep_spec))
-                {
+    //            {
-                    (*it)->state = InstallState::NEEDS_REBUILD;
+    //                (*it)->state = InstallState::NEEDS_REBUILD;
-                    return true;
+    //                return true;
-                }
+    //            }
-            }
+    //        }
 
-            return (*it)->needs_rebuild();
+    //        return (*it)->needs_rebuild();
-        }
+    //    }
 
-        return false;
+    //    return false;
-    }
+    //}
 
     StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr<StatusParagraph> pgh)
     {