From 459908ae14458a615bb0a8c278c799cabd34e558 Mon Sep 17 00:00:00 2001 From: "Curtis.Bezault" Date: Tue, 23 Jul 2019 10:07:39 -0700 Subject: [PATCH] add external file hashes to the binary paragraph --- scripts/get_triplet_environment.cmake | 2 +- toolsrc/include/vcpkg/binaryparagraph.h | 1 + toolsrc/include/vcpkg/build.h | 6 +- toolsrc/src/vcpkg/binaryparagraph.cpp | 34 ++++- toolsrc/src/vcpkg/build.cpp | 157 ++++++++++++++++-------- toolsrc/src/vcpkg/statusparagraphs.cpp | 42 +++---- 6 files changed, 161 insertions(+), 81 deletions(-) diff --git a/scripts/get_triplet_environment.cmake b/scripts/get_triplet_environment.cmake index ab5e8f8c3..fa1e5cb39 100644 --- a/scripts/get_triplet_environment.cmake +++ b/scripts/get_triplet_environment.cmake @@ -13,4 +13,4 @@ message("VCPKG_VISUAL_STUDIO_PATH=${VCPKG_VISUAL_STUDIO_PATH}") message("VCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}") message("VCPKG_BUILD_TYPE=${VCPKG_BUILD_TYPE}") message("VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}") -message("VCPKG_ABI_ADDITIONAL_FILES=${VCPKG_ABI_ADDITIONAL_FILES}") +message("VCPKG_EXTERNAL_FILES=${VCPKG_EXTERNAL_FILES}") diff --git a/toolsrc/include/vcpkg/binaryparagraph.h b/toolsrc/include/vcpkg/binaryparagraph.h index fa49edaba..ec14f8a97 100644 --- a/toolsrc/include/vcpkg/binaryparagraph.h +++ b/toolsrc/include/vcpkg/binaryparagraph.h @@ -32,6 +32,7 @@ namespace vcpkg std::vector depends; std::string abi; SourceParagraph::TYPE type; + std::unordered_map external_files; }; struct BinaryControlFile diff --git a/toolsrc/include/vcpkg/build.h b/toolsrc/include/vcpkg/build.h index 85d6ebed9..5b52f436f 100644 --- a/toolsrc/include/vcpkg/build.h +++ b/toolsrc/include/vcpkg/build.h @@ -138,7 +138,7 @@ namespace vcpkg::Build Optional external_toolchain_file; Optional build_type; std::vector passthrough_env_vars; - std::vector additional_files; + std::vector> external_files; }; std::string make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset); @@ -153,7 +153,7 @@ namespace vcpkg::Build CHAINLOAD_TOOLCHAIN_FILE, BUILD_TYPE, ENV_PASSTHROUGH, - ABI_ADDITIONAL_FILES, + EXTERNAL_FILES, }; const std::unordered_map VCPKG_OPTIONS = { @@ -165,7 +165,7 @@ namespace vcpkg::Build {"VCPKG_CHAINLOAD_TOOLCHAIN_FILE", VcpkgTripletVar::CHAINLOAD_TOOLCHAIN_FILE}, {"VCPKG_BUILD_TYPE", VcpkgTripletVar::BUILD_TYPE}, {"VCPKG_ENV_PASSTHROUGH", VcpkgTripletVar::ENV_PASSTHROUGH}, - {"VCPKG_ABI_ADDITIONAL_FILES", VcpkgTripletVar::ABI_ADDITIONAL_FILES}, + {"VCPKG_EXTERNAL_FILES", VcpkgTripletVar::EXTERNAL_FILES}, }; struct ExtendedBuildResult diff --git a/toolsrc/src/vcpkg/binaryparagraph.cpp b/toolsrc/src/vcpkg/binaryparagraph.cpp index ad7790fe1..2e9415dab 100644 --- a/toolsrc/src/vcpkg/binaryparagraph.cpp +++ b/toolsrc/src/vcpkg/binaryparagraph.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -23,7 +24,7 @@ namespace vcpkg static const std::string MAINTAINER = "Maintainer"; static const std::string DEPENDS = "Depends"; static const std::string DEFAULTFEATURES = "Default-Features"; - static const std::string TYPE = "Type"; + static const std::string EXTERNALFILES = "External-Files"; } BinaryParagraph::BinaryParagraph() = default; @@ -61,8 +62,23 @@ namespace vcpkg this->default_features = parse_comma_list(parser.optional_field(Fields::DEFAULTFEATURES)); } - this->type = - SourceParagraph::type_from_string(parser.optional_field(Fields::TYPE)); + std::vector external_files_or_hashes = + parse_comma_list(parser.optional_field(Fields::EXTERNALFILES)); + + if (external_files_or_hashes.size() % 2 != 0) + { + Checks::exit_with_message( + VCPKG_LINE_INFO, + "The External-Files field is not composed of key-value pairs for ", + this->spec); + } + + for (int i = 0; i < external_files_or_hashes.size(); i += 2) + { + external_files.emplace( + std::move(external_files_or_hashes[i]), + std::move(external_files_or_hashes[i+1])); + } if (const auto err = parser.error_info(this->spec.to_string())) { @@ -124,6 +140,16 @@ namespace vcpkg if (!pgh.abi.empty()) out_str.append("Abi: ").append(pgh.abi).push_back('\n'); if (!pgh.description.empty()) out_str.append("Description: ").append(pgh.description).push_back('\n'); - out_str.append("Type: ").append(SourceParagraph::string_from_type(pgh.type)).push_back('\n'); + if (!pgh.external_files.empty()) + { + out_str.append("External-Files: "); + out_str.append(Strings::join(",", + Util::fmap( + pgh.external_files, + [](const std::pair& kv) + { + return kv.first + "," + kv.second; + }))).push_back('\n'); + } } } diff --git a/toolsrc/src/vcpkg/build.cpp b/toolsrc/src/vcpkg/build.cpp index 47599bfca..110a571b0 100644 --- a/toolsrc/src/vcpkg/build.cpp +++ b/toolsrc/src/vcpkg/build.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -261,10 +262,12 @@ namespace vcpkg::Build return BinaryParagraph(source_paragraph, feature_paragraph, triplet); } - static std::unique_ptr create_binary_control_file(const SourceParagraph& source_paragraph, - const Triplet& triplet, - const BuildInfo& build_info, - const std::string& abi_tag) + static std::unique_ptr create_binary_control_file( + const SourceParagraph& source_paragraph, + const Triplet& triplet, + const BuildInfo& build_info, + const PreBuildInfo& pre_build_info, + const std::string& abi_tag) { auto bcf = std::make_unique(); BinaryParagraph bpgh(source_paragraph, triplet, abi_tag); @@ -272,6 +275,14 @@ namespace vcpkg::Build { bpgh.version = *p_ver; } + + for (auto& file_hash : pre_build_info.external_files) + { + bpgh.external_files.emplace( + std::move(file_hash.first), + std::move(file_hash.second)); + } + bcf->core_paragraph = std::move(bpgh); return bcf; } @@ -447,6 +458,41 @@ namespace vcpkg::Build return command; } + static std::vector> get_external_file_hashes( + const VcpkgPaths& paths, + const std::vector& files) + { + static std::map s_hash_cache; + + const auto& fs = paths.get_filesystem(); + + std::vector> hashes; + for (const fs::path& external_file : files) + { + auto it_hash = s_hash_cache.find(external_file); + + if (it_hash != s_hash_cache.end()) + { + hashes.emplace_back(external_file, it_hash->second); + } + else if (fs.is_regular_file(external_file)) + { + auto emp = s_hash_cache.emplace(external_file.u8string(), + Hash::get_file_hash(fs, external_file, "SHA1")); + hashes.emplace_back(external_file, emp.first->second); + } + else + { + Checks::exit_with_message( + VCPKG_LINE_INFO, + external_file.u8string() + + " was listed as an additional file for calculating the abi, but was not found."); + } + } + + return hashes; + } + static std::string get_triplet_abi(const VcpkgPaths& paths, const PreBuildInfo& pre_build_info, const Triplet& triplet) @@ -496,31 +542,6 @@ namespace vcpkg::Build s_hash_cache.emplace(triplet_file_path, hash); } - for (const fs::path& additional_file : pre_build_info.additional_files) - { - it_hash = s_hash_cache.find(additional_file); - - if (it_hash != s_hash_cache.end()) - { - hash += "-"; - hash += it_hash->second; - } - else if (fs.is_regular_file(additional_file)) - { - std::string tmp_hash = Hash::get_file_hash(fs, additional_file, "SHA1"); - hash += "-"; - hash += tmp_hash; - - s_hash_cache.emplace(additional_file, tmp_hash); - } - else - { - Checks::exit_with_message( - VCPKG_LINE_INFO, - additional_file.u8string() + " was listed as an additional file for calculating the abi, but was not found."); - } - } - return hash; } @@ -572,7 +593,13 @@ namespace vcpkg::Build const size_t error_count = PostBuildLint::perform_all_checks(spec, paths, pre_build_info, build_info, config.port_dir); - auto bcf = create_binary_control_file(*config.scf.core_paragraph, triplet, build_info, abi_tag); + std::unique_ptr bcf = + create_binary_control_file( + *config.scf.core_paragraph, + triplet, + build_info, + pre_build_info, + abi_tag); if (error_count != 0) { @@ -629,7 +656,9 @@ namespace vcpkg::Build std::vector abi_tag_entries(dependency_abis.begin(), dependency_abis.end()); - abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); + // Sorted here as the order of dependency_abis is the only + // non-deterministicly ordered set of AbiEntries + Util::sort(abi_tag_entries); // If there is an unusually large number of files in the port then // something suspicious is going on. Rather than hash all of them @@ -637,13 +666,16 @@ namespace vcpkg::Build const int max_port_file_count = 100; // the order of recursive_directory_iterator is undefined so save the names to sort - std::vector port_files; + std::vector> hashes_files; for (auto& port_file : fs::stdfs::recursive_directory_iterator(config.port_dir)) { if (fs::is_regular_file(status(port_file))) { - port_files.push_back(port_file); - if (port_files.size() > max_port_file_count) + hashes_files.emplace_back( + vcpkg::Hash::get_file_hash(fs, port_file, "SHA1"), + port_file.path().filename()); + + if (hashes_files.size() > max_port_file_count) { abi_tag_entries.emplace_back(AbiEntry{"no_hash_max_portfile", ""}); break; @@ -651,25 +683,45 @@ namespace vcpkg::Build } } - if (port_files.size() <= max_port_file_count) + if (hashes_files.size() <= max_port_file_count) { - std::sort(port_files.begin(), port_files.end()); + Util::sort(hashes_files); - int counter = 0; - for (auto& port_file : port_files) + for (auto& hash_file : hashes_files) { - // When vcpkg takes a dependency on C++17 it can use fs::relative, - // which will give a stable ordering and better names in the key entry. - // this is not available in the filesystem TS so instead number the files for the key. - std::string key = Strings::format("file_%03d", counter++); - if (Debug::g_debugging) - { - System::print2("[DEBUG] mapping ", key, " from ", port_file.u8string(), "\n"); - } - abi_tag_entries.emplace_back(AbiEntry{key, vcpkg::Hash::get_file_hash(fs, port_file, "SHA1")}); + // We've already sorted by hash so it's safe to write down the + // filename, which will be consistent across machines. + abi_tag_entries.emplace_back( + AbiEntry{ + std::move(hash_file.second), + std::move(hash_file.first) + }); } } + //Make a copy of the external files and their hashes, and sort by hash + auto additional_file_hashes = pre_build_info.external_files; + std::sort( + additional_file_hashes.begin(), + additional_file_hashes.end(), + [](const std::pair& l, + const std::pair& r) + { + return l.second < r.second || + (l.second == r.second && l.first < r.first); + }); + + for (auto& hash_file : additional_file_hashes) + { + abi_tag_entries.emplace_back( + AbiEntry{ + std::move(hash_file.first), + std::move(hash_file.second) + }); + } + + abi_tag_entries.emplace_back(AbiEntry{"cmake", paths.get_tool_version(Tools::CMAKE)}); + abi_tag_entries.emplace_back(AbiEntry{ "vcpkg_fixup_cmake_targets", vcpkg::Hash::get_file_hash(fs, paths.scripts / "cmake" / "vcpkg_fixup_cmake_targets.cmake", "SHA1")}); @@ -682,8 +734,6 @@ namespace vcpkg::Build if (config.build_package_options.use_head_version == UseHeadVersion::YES) abi_tag_entries.emplace_back(AbiEntry{"head", ""}); - Util::sort(abi_tag_entries); - const std::string full_abi_info = Strings::join("", abi_tag_entries, [](const AbiEntry& p) { return p.key + " " + p.value + "\n"; }); @@ -1114,12 +1164,15 @@ namespace vcpkg::Build case VcpkgTripletVar::ENV_PASSTHROUGH : pre_build_info.passthrough_env_vars = Strings::split(variable_value, ";"); break; - case VcpkgTripletVar::ABI_ADDITIONAL_FILES : - pre_build_info.additional_files = Util::fmap(Strings::split(variable_value, ";"), + case VcpkgTripletVar::EXTERNAL_FILES : + pre_build_info.external_files = + get_external_file_hashes( + paths, + Util::fmap(Strings::split(variable_value, ";"), [](const std::string& path) { return fs::path{path}; - }); + })); break; } } diff --git a/toolsrc/src/vcpkg/statusparagraphs.cpp b/toolsrc/src/vcpkg/statusparagraphs.cpp index f204ed568..4f01b9918 100644 --- a/toolsrc/src/vcpkg/statusparagraphs.cpp +++ b/toolsrc/src/vcpkg/statusparagraphs.cpp @@ -118,30 +118,30 @@ namespace vcpkg return it != end() && (*it)->is_installed(); } - bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec) - { - auto it = find(spec); - if (it != end()) - { - for (const std::string& dep : (*it)->package.depends) - { - PackageSpec dep_spec = - PackageSpec::from_name_and_triplet( - dep, - spec.triplet()).value_or_exit(VCPKG_LINE_INFO); + //bool vcpkg::StatusParagraphs::needs_rebuild(const PackageSpec& spec) + //{ + // auto it = find(spec); + // if (it != end()) + // { + // for (const std::string& dep : (*it)->package.depends) + // { + // PackageSpec dep_spec = + // PackageSpec::from_name_and_triplet( + // dep, + // spec.triplet()).value_or_exit(VCPKG_LINE_INFO); - if (needs_rebuild(dep_spec)) - { - (*it)->state = InstallState::NEEDS_REBUILD; - return true; - } - } + // if (needs_rebuild(dep_spec)) + // { + // (*it)->state = InstallState::NEEDS_REBUILD; + // return true; + // } + // } - return (*it)->needs_rebuild(); - } + // return (*it)->needs_rebuild(); + // } - return false; - } + // return false; + //} StatusParagraphs::iterator StatusParagraphs::insert(std::unique_ptr pgh) {