[vcpkg] Implementation of --x-binarysource=nuget (and friends) (#12058)

* [vcpkg] Initial implementation of --x-binarysource=nuget

* [vcpkg] Remove double-double quoting of CMake arguments

* [vcpkg] Update nuget.exe to 5.5.1 to support Azure DevOps Artifacts

* [vcpkg] Enable batching of NuGet server calls with prefetch(). Add `interactive` binarysource.

* [vcpkg] Add `nugetconfig` binary source

* [vcpkg] Short circuit querying remote NuGet servers once all refs are found

* [vcpkg] Add experimental help for binary caching

* [vcpkg] Improved NuGet cache package descriptions and version formatting

* [vcpkg] Rename `CmdLineBuilder::build()` to extract()

* [vcpkg-help] Ascii-betize help topics

* [vcpkg] Add tests for cmdlinebuilder. Improve handling of quotes and slashes.

* [vcpkg] Addressing code review comments

* [vcpkg] Add tests for vcpkg::reformat_version()

* [vcpkg] Added test for vcpkg::generate_nuspec()

* [vcpkg] Add tests for vcpkg::XmlSerializer

* [vcpkg] Addressed code review comment

* [vcpkg] Add test for vcpkg::Strings::find_first_of

* [vcpkg] Fix machine-specific paths in test for vcpkg::generate_nuspec()

Co-authored-by: Robert Schumacher <roschuma@microsoft.com>

scripts/vcpkgTools.xml

@@ -60,10 +60,10 @@
         <sha512>f477842d0cebefcd6bf9c6d536ab8ea20ec5b0aa967ee963ab6a101aeff9df8742ca600d35f39e2e7158d76d8231f1ed2bef6104dce84d2bf8d6b07d17d706a1</sha512>
     </tool>
     <tool name="nuget" os="windows">
-        <version>4.8.1</version>
+        <version>5.5.1</version>
         <exeRelativePath>nuget.exe</exeRelativePath>
-        <url>https://dist.nuget.org/win-x86-commandline/v4.8.1/nuget.exe</url>
-        <sha512>42cb744338af8decc033a75bce5b4c4df28e102bafc45f9a8ba86d7bc010f5b43ebacae80d7b28c4f85ac900eefc2a349620ae65f27f6ca1c21c53b63b92924b</sha512>
+        <url>https://dist.nuget.org/win-x86-commandline/v5.5.1/nuget.exe</url>
+        <sha512>22ea847d8017cd977664d0b13c889cfb13c89143212899a511be217345a4e243d4d8d4099700114a11d26a087e83eb1a3e2b03bdb5e0db48f10403184cd26619</sha512>
     </tool>
     <tool name="installerbase" os="windows">
         <version>3.1.81</version>

toolsrc/include/vcpkg/base/strings.h

@@ -175,6 +175,8 @@ namespace vcpkg::Strings
 
     std::vector<std::string> split(const std::string& s, const char delimiter);
 
+    const char* find_first_of(StringView searched, StringView candidates);
+
     std::vector<StringView> find_all_enclosed(StringView input, StringView left_delim, StringView right_delim);
 
     StringView find_exactly_one_enclosed(StringView input, StringView left_tag, StringView right_tag);

toolsrc/include/vcpkg/base/system.process.h

@@ -23,6 +23,18 @@ namespace vcpkg::System
                                      const fs::path& cmake_script,
                                      const std::vector<CMakeVariable>& pass_variables);
 
+    struct CmdLineBuilder
+    {
+        CmdLineBuilder& path_arg(const fs::path& p) { return string_arg(p.u8string()); }
+        CmdLineBuilder& string_arg(StringView s);
+        std::string extract() noexcept { return std::move(buf); }
+
+        operator ZStringView() const { return buf; }
+
+    private:
+        std::string buf;
+    };
+
     fs::path get_exe_path_of_current_process();
 
     struct ExitCodeAndOutput

toolsrc/include/vcpkg/base/util.h

@@ -49,6 +49,19 @@ namespace vcpkg::Util
         }
     }
 
+    template<class Range, class Pred, class E = ElementT<Range>>
+    std::vector<E> filter(const Range& xs, Pred&& f)
+    {
+        std::vector<E> ret;
+
+        for (auto&& x : xs)
+        {
+            if (f(x)) ret.push_back(x);
+        }
+
+        return ret;
+    }
+
     template<class Range, class Func>
     using FmapOut = std::remove_reference_t<decltype(std::declval<Func&>()(*std::declval<Range>().begin()))>;
 

toolsrc/include/vcpkg/binarycaching.h

@@ -8,6 +8,7 @@
 namespace vcpkg::Dependencies
 {
     struct InstallPlanAction;
+    struct ActionPlan;
 }
 namespace vcpkg::Build
 {
@@ -27,10 +28,17 @@ namespace vcpkg
     struct IBinaryProvider
     {
         virtual ~IBinaryProvider() = default;
-        virtual void prefetch() = 0;
+        /// Gives the BinaryProvider an opportunity to batch any downloading or server communication for executing
+        /// `plan`.
+        virtual void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) = 0;
+        /// Attempts to restore the package referenced by `action` into the packages directory.
         virtual RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+        /// Called upon a successful build of `action`
         virtual void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) = 0;
+        /// Called upon a failure during the build of `action`
         virtual void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) = 0;
+        /// Requests the result of `try_restore()` without actually downloading the package. Used by CI to determine
+        /// missing packages.
         virtual RestoreResult precheck(const VcpkgPaths& paths,
                                        const Dependencies::InstallPlanAction& action,
                                        bool purge_tombstones) = 0;
@@ -42,4 +50,6 @@ namespace vcpkg
                                                                                     View<std::string> args);
     ExpectedS<std::unique_ptr<IBinaryProvider>> create_binary_provider_from_configs_pure(const std::string& env_string,
                                                                                          View<std::string> args);
+
+    void help_topic_binary_caching(const VcpkgPaths& paths);
 }

toolsrc/include/vcpkg/binarycaching.private.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <string>
+#include <vcpkg/dependencies.h>
+#include <vcpkg/packagespec.h>
+#include <vcpkg/vcpkgpaths.h>
+
+namespace vcpkg
+{
+    std::string reformat_version(const std::string& version, const std::string& abi_tag);
+
+    struct NugetReference
+    {
+        explicit NugetReference(const Dependencies::InstallPlanAction& action)
+            : NugetReference(action.spec,
+                             action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO)
+                                 .source_control_file->core_paragraph->version,
+                             action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi)
+        {
+        }
+
+        NugetReference(const PackageSpec& spec, const std::string& raw_version, const std::string& abi_tag)
+            : id(spec.dir()), version(reformat_version(raw_version, abi_tag))
+        {
+        }
+
+        std::string id;
+        std::string version;
+
+        std::string nupkg_filename() const { return Strings::concat(id, '.', version, ".nupkg"); }
+    };
+
+    std::string generate_nuspec(const VcpkgPaths& paths,
+                                const Dependencies::InstallPlanAction& action,
+                                const NugetReference& ref);
+
+    struct XmlSerializer
+    {
+        std::string buf;
+        int indent = 0;
+
+        XmlSerializer& emit_declaration();
+        XmlSerializer& open_tag(StringLiteral sl);
+        XmlSerializer& start_complex_open_tag(StringLiteral sl);
+        XmlSerializer& text_attr(StringLiteral name, StringView content);
+        XmlSerializer& finish_complex_open_tag();
+        XmlSerializer& finish_self_closing_complex_tag();
+        XmlSerializer& close_tag(StringLiteral sl);
+        XmlSerializer& text(StringView sv);
+        XmlSerializer& simple_tag(StringLiteral tag, StringView content);
+        XmlSerializer& line_break();
+    };
+
+}

toolsrc/include/vcpkg/build.h

@@ -284,7 +284,8 @@ namespace vcpkg::Build
     struct AbiInfo
     {
         std::unique_ptr<PreBuildInfo> pre_build_info;
-        const Toolset* toolset;
+        Optional<const Toolset&> toolset;
+        Optional<const std::string&> triplet_abi;
         std::string package_abi;
         Optional<fs::path> abi_tag_file;
     };

toolsrc/include/vcpkg/vcpkgcmdarguments.h

@@ -92,6 +92,7 @@ namespace vcpkg
         void example(StringView example_text);
         void header(StringView name);
         void blank();
+        void text(StringView text, int indent = 0);
 
         std::string m_str;
     };

toolsrc/src/vcpkg-test/binarycaching.cpp

@@ -0,0 +1,138 @@
+#include <catch2/catch.hpp>
+#include <vcpkg/binarycaching.private.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/vcpkgcmdarguments.h>
+#include <vcpkg/sourceparagraph.h>
+#include <vcpkg/paragraphs.h>
+#include <vcpkg/dependencies.h>
+#include <string>
+
+using namespace vcpkg;
+
+TEST_CASE ("reformat_version semver-ish", "[reformat_version]")
+{
+    REQUIRE(reformat_version("0.0.0", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("1.0.1", "abitag") == "1.0.1-abitag");
+    REQUIRE(reformat_version("1.01.000", "abitag") == "1.1.0-abitag");
+    REQUIRE(reformat_version("1.2", "abitag") == "1.2.0-abitag");
+    REQUIRE(reformat_version("v52", "abitag") == "52.0.0-abitag");
+    REQUIRE(reformat_version("v09.01.02", "abitag") == "9.1.2-abitag");
+    REQUIRE(reformat_version("1.1.1q", "abitag") == "1.1.1-abitag");
+    REQUIRE(reformat_version("1", "abitag") == "1.0.0-abitag");
+}
+
+TEST_CASE ("reformat_version date", "[reformat_version]")
+{
+    REQUIRE(reformat_version("2020-06-26", "abitag") == "2020.6.26-abitag");
+    REQUIRE(reformat_version("20-06-26", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("2020-06-26-release", "abitag") == "2020.6.26-abitag");
+    REQUIRE(reformat_version("2020-06-26000", "abitag") == "2020.6.26-abitag");
+}
+
+TEST_CASE ("reformat_version generic", "[reformat_version]")
+{
+    REQUIRE(reformat_version("apr", "abitag") == "0.0.0-abitag");
+    REQUIRE(reformat_version("", "abitag") == "0.0.0-abitag");
+}
+
+TEST_CASE ("generate_nuspec", "[generate_nuspec]")
+{
+    auto& fsWrapper = Files::get_real_filesystem();
+    VcpkgCmdArguments args = VcpkgCmdArguments::create_from_arg_sequence(nullptr, nullptr);
+    args.packages_root_dir = std::make_unique<std::string>("/");
+    VcpkgPaths paths(fsWrapper, args);
+
+    auto pghs = Paragraphs::parse_paragraphs(R"(
+Source: zlib2
+Version: 1.5
+Build-Depends: zlib
+Description: a spiffy compression library wrapper
+
+Feature: a
+Description: a feature
+
+Feature: b
+Description: enable bzip capabilities
+Build-Depends: bzip
+)",
+                                             "<testdata>");
+    REQUIRE(pghs.has_value());
+    auto maybe_scf = SourceControlFile::parse_control_file(fs::path(), std::move(*pghs.get()));
+    REQUIRE(maybe_scf.has_value());
+    SourceControlFileLocation scfl{std::move(*maybe_scf.get()), fs::path()};
+
+    Dependencies::InstallPlanAction ipa(PackageSpec{"zlib2", Triplet::X64_WINDOWS},
+                                        scfl,
+                                        Dependencies::RequestType::USER_REQUESTED,
+                                        {{"a", {}}, {"b", {}}});
+
+    ipa.abi_info = Build::AbiInfo{};
+    ipa.abi_info.get()->package_abi = "packageabi";
+    std::string tripletabi("tripletabi");
+    ipa.abi_info.get()->triplet_abi = tripletabi;
+
+    NugetReference ref(ipa);
+
+    REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-packageabi.nupkg");
+
+    auto nuspec = generate_nuspec(paths, ipa, ref);
+#ifdef _WIN32
+#define PKGPATH "C:\\zlib2_x64-windows\\**"
+#else
+#define PKGPATH "/zlib2_x64-windows/**"
+#endif
+    std::string expected = R"(<package>
+  <metadata>
+    <id>zlib2_x64-windows</id>
+    <version>1.5.0-packageabi</version>
+    <authors>vcpkg</authors>
+    <description>NOT FOR DIRECT USE. Automatically generated cache package.
+
+a spiffy compression library wrapper
+
+Version: 1.5
+Triplet/Compiler hash: tripletabi
+Features: a, b
+Dependencies:
+</description>
+    <packageTypes><packageType name="vcpkg"/></packageTypes>
+    </metadata>
+  <files><file src=")" PKGPATH R"(" target=""/></files>
+  </package>
+)";
+    auto expected_lines = Strings::split(expected, '\n');
+    auto nuspec_lines = Strings::split(nuspec, '\n');
+    for (size_t i = 0; i < expected_lines.size() && i < nuspec_lines.size(); ++i)
+    {
+        INFO("on line: " << i);
+        REQUIRE(nuspec_lines[i] == expected_lines[i]);
+    }
+    REQUIRE(nuspec_lines.size() == expected_lines.size());
+}
+
+TEST_CASE ("XmlSerializer", "[XmlSerializer]")
+{
+    XmlSerializer xml;
+    xml.open_tag("a");
+    xml.open_tag("b");
+    xml.simple_tag("c", "d");
+    xml.close_tag("b");
+    xml.text("escaping: & < > \" '");
+
+    REQUIRE(xml.buf == R"(<a><b><c>d</c></b>escaping: &amp; &lt; &gt; &quot; &apos;)");
+
+    xml = XmlSerializer();
+    xml.emit_declaration();
+    xml.start_complex_open_tag("a").text_attr("b", "<").text_attr("c", "  ").finish_self_closing_complex_tag();
+    REQUIRE(xml.buf == R"(<?xml version="1.0" encoding="utf-8"?><a b="&lt;" c="  "/>)");
+
+    xml = XmlSerializer();
+    xml.start_complex_open_tag("a").finish_complex_open_tag();
+    REQUIRE(xml.buf == R"(<a>)");
+
+    xml = XmlSerializer();
+    xml.line_break();
+    xml.open_tag("a").line_break().line_break();
+    xml.close_tag("a").line_break().line_break();
+    REQUIRE(xml.buf == "\n<a>\n  \n  </a>\n\n");
+}

toolsrc/src/vcpkg-test/binaryconfigparser.cpp

@@ -57,6 +57,78 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]")
     }
 }
 
+TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget,relative-path", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget,http://example.org/", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH, {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",nonsense", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",upload", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget," ABSOLUTE_PATH ",upload,extra", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nuget,,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
+TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig,relative-path", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig,http://example.org/", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH, {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",nonsense", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",upload", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",upload,extra", {});
+        REQUIRE(!parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("nugetconfig,,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
 TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]")
 {
     {
@@ -89,6 +161,18 @@ TEST_CASE ("BinaryConfigParser clear provider", "[binaryconfigparser]")
     }
 }
 
+TEST_CASE ("BinaryConfigParser interactive provider", "[binaryconfigparser]")
+{
+    {
+        auto parsed = create_binary_provider_from_configs_pure("interactive", {});
+        REQUIRE(parsed.has_value());
+    }
+    {
+        auto parsed = create_binary_provider_from_configs_pure("interactive,upload", {});
+        REQUIRE(!parsed.has_value());
+    }
+}
+
 TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]")
 {
     {

toolsrc/src/vcpkg-test/strings.cpp

@@ -41,3 +41,13 @@ TEST_CASE ("split by char", "[strings]")
     REQUIRE(split("    hello  world    ", ' ') == result_t{"hello", "world"});
     REQUIRE(split("no delimiters", ',') == result_t{"no delimiters"});
 }
+
+TEST_CASE ("find_first_of", "[strings]")
+{
+    using vcpkg::Strings::find_first_of;
+    REQUIRE(find_first_of("abcdefg", "hij") == std::string());
+    REQUIRE(find_first_of("abcdefg", "a") == std::string("abcdefg"));
+    REQUIRE(find_first_of("abcdefg", "g") == std::string("g"));
+    REQUIRE(find_first_of("abcdefg", "bg") == std::string("bcdefg"));
+    REQUIRE(find_first_of("abcdefg", "gb") == std::string("bcdefg"));
+}

toolsrc/src/vcpkg-test/system.cpp

@@ -7,16 +7,17 @@
 #include <vcpkg/base/zstringview.h>
 #include <vcpkg/base/strings.h>
 #include <vcpkg/base/system.h>
+#include <vcpkg/base/system.process.h>
 
+using vcpkg::nullopt;
 using vcpkg::Optional;
 using vcpkg::StringView;
 using vcpkg::ZStringView;
 using vcpkg::Checks::check_exit;
-using vcpkg::System::get_environment_variable;
-using vcpkg::System::to_cpu_architecture;
-using vcpkg::System::guess_visual_studio_prompt_target_architecture;
-using vcpkg::nullopt;
 using vcpkg::System::CPUArchitecture;
+using vcpkg::System::get_environment_variable;
+using vcpkg::System::guess_visual_studio_prompt_target_architecture;
+using vcpkg::System::to_cpu_architecture;
 
 namespace
 {
@@ -126,30 +127,43 @@ TEST_CASE ("guess_visual_studio_prompt", "[system]")
     set_environment_variable("VSCMD_ARG_TGT_ARCH", nullopt);
     CHECK(!guess_visual_studio_prompt_target_architecture().has_value());
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "x86");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "x64");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "arm");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", "arm64");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
 
     // check that apparent "nested" prompts defer to "vsdevcmd"
     set_environment_variable("VCINSTALLDIR", "anything");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::ARM64);
     set_environment_variable("VSCMD_ARG_TGT_ARCH", nullopt);
     set_environment_variable("Platform", nullopt);
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
     set_environment_variable("Platform", "x86");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X86);
     set_environment_variable("Platform", "x64");
-    CHECK(guess_visual_studio_prompt_target_architecture()
-        .value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
+    CHECK(guess_visual_studio_prompt_target_architecture().value_or_exit(VCPKG_LINE_INFO) == CPUArchitecture::X64);
+}
+
+TEST_CASE ("cmdlinebuilder", "[system]")
+{
+    using vcpkg::System::CmdLineBuilder;
+
+    CmdLineBuilder cmd;
+    cmd.path_arg(fs::u8path("relative/path.exe"));
+    cmd.string_arg("abc");
+    cmd.string_arg("hello world!");
+    cmd.string_arg("|");
+    cmd.string_arg(";");
+    REQUIRE(cmd.extract() == "relative/path.exe abc \"hello world!\" \"|\" \";\"");
+
+    cmd.path_arg(fs::u8path("trailing\\slash\\"));
+    cmd.string_arg("inner\"quotes");
+#ifdef _WIN32
+    REQUIRE(cmd.extract() == "\"trailing\\slash\\\\\" \"inner\\\"quotes\"");
+#else
+    REQUIRE(cmd.extract() == "\"trailing\\\\slash\\\\\" \"inner\\\"quotes\"");
+#endif
 }

toolsrc/src/vcpkg/base/strings.cpp

@@ -176,6 +176,11 @@ std::vector<std::string> Strings::split(const std::string& s, const char delimit
     }
 }
 
+const char* Strings::find_first_of(StringView input, StringView chars)
+{
+    return std::find_first_of(input.begin(), input.end(), chars.begin(), chars.end());
+}
+
 std::vector<StringView> Strings::find_all_enclosed(StringView input, StringView left_delim, StringView right_delim)
 {
     auto it_left = input.begin();

toolsrc/src/vcpkg/base/system.process.cpp

@@ -30,7 +30,7 @@ namespace vcpkg
     {
         struct CtrlCStateMachine
         {
-            CtrlCStateMachine() : m_number_of_external_processes(0), m_global_job(NULL), m_in_interactive(0) { }
+            CtrlCStateMachine() : m_number_of_external_processes(0), m_global_job(NULL), m_in_interactive(0) {}
 
             void transition_to_spawn_process() noexcept
             {
@@ -167,7 +167,7 @@ namespace vcpkg
     }
 
     System::CMakeVariable::CMakeVariable(const StringView varname, const char* varvalue)
-        : s(Strings::format(R"("-D%s=%s")", varname, varvalue))
+        : s(Strings::format("-D%s=%s", varname, varvalue))
     {
     }
     System::CMakeVariable::CMakeVariable(const StringView varname, const std::string& varvalue)
@@ -183,10 +183,62 @@ namespace vcpkg
                                              const fs::path& cmake_script,
                                              const std::vector<CMakeVariable>& pass_variables)
     {
-        return Strings::format(R"("%s" %s -P "%s")",
-                               cmake_tool_path.u8string(),
-                               Strings::join(" ", pass_variables, [](auto&& v) { return v.s; }),
-                               cmake_script.generic_u8string());
+        System::CmdLineBuilder cmd;
+        cmd.path_arg(cmake_tool_path);
+        for (auto&& var : pass_variables)
+        {
+            cmd.string_arg(var.s);
+        }
+        cmd.string_arg("-P").path_arg(cmake_script);
+        return cmd.extract();
+    }
+
+    System::CmdLineBuilder& System::CmdLineBuilder::string_arg(StringView s)
+    {
+        if (!buf.empty()) buf.push_back(' ');
+        if (Strings::find_first_of(s, " \t\n\r\"\\,;&`^|'") != s.end())
+        {
+            // TODO: improve this to properly handle all escaping
+#if _WIN32
+            // On Windows, `\`s before a double-quote must be doubled. Inner double-quotes must be escaped.
+            buf.push_back('"');
+            size_t n_slashes = 0;
+            for (auto ch : s)
+            {
+                if (ch == '\\')
+                {
+                    ++n_slashes;
+                }
+                else if (ch == '"')
+                {
+                    buf.append(n_slashes + 1, '\\');
+                    n_slashes = 0;
+                }
+                else
+                {
+                    n_slashes = 0;
+                }
+                buf.push_back(ch);
+            }
+            buf.append(n_slashes, '\\');
+            buf.push_back('"');
+#else
+            // On non-Windows, `\` is the escape character and always requires doubling. Inner double-quotes must be
+            // escaped.
+            buf.push_back('"');
+            for (auto ch : s)
+            {
+                if (ch == '\\' || ch == '"') buf.push_back('\\');
+                buf.push_back(ch);
+            }
+            buf.push_back('"');
+#endif
+        }
+        else
+        {
+            Strings::append(buf, s);
+        }
+        return *this;
     }
 
 #if defined(_WIN32)
@@ -330,7 +382,7 @@ namespace vcpkg
 #if defined(_WIN32)
     struct ProcessInfo
     {
-        constexpr ProcessInfo() noexcept : proc_info{} { }
+        constexpr ProcessInfo() noexcept : proc_info{} {}
         ProcessInfo(ProcessInfo&& other) noexcept : proc_info(other.proc_info)
         {
             other.proc_info.hProcess = nullptr;
@@ -672,6 +724,6 @@ namespace vcpkg
         SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(ctrl_handler), TRUE);
     }
 #else
-    void System::register_console_ctrl_handler() { }
+    void System::register_console_ctrl_handler() {}
 #endif
 }

toolsrc/src/vcpkg/binarycaching.cpp

@@ -3,9 +3,11 @@
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/files.h>
 #include <vcpkg/base/parse.h>
+#include <vcpkg/base/system.debug.h>
 #include <vcpkg/base/system.print.h>
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/binarycaching.h>
+#include <vcpkg/binarycaching.private.h>
 #include <vcpkg/build.h>
 #include <vcpkg/dependencies.h>
 
@@ -66,7 +68,7 @@ namespace
         {
         }
         ~ArchivesBinaryProvider() = default;
-        void prefetch() override {}
+        void prefetch(const VcpkgPaths&, const Dependencies::ActionPlan&) override {}
         RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
         {
             const auto& abi_tag = action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi;
@@ -244,12 +246,361 @@ namespace
             return RestoreResult::missing;
         }
 
+    private:
         std::vector<fs::path> m_read_dirs, m_write_dirs;
     };
 
+    static std::string trim_leading_zeroes(std::string v)
+    {
+        auto n = v.find_first_not_of('0');
+        if (n == std::string::npos)
+        {
+            v = "0";
+        }
+        else if (n > 0)
+        {
+            v.erase(0, n);
+        }
+        return v;
+    }
+
+    struct NugetBinaryProvider : IBinaryProvider
+    {
+        NugetBinaryProvider(std::vector<std::string>&& read_sources,
+                            std::vector<std::string>&& write_sources,
+                            std::vector<fs::path>&& read_configs,
+                            std::vector<fs::path>&& write_configs,
+                            bool interactive)
+            : m_read_sources(std::move(read_sources))
+            , m_write_sources(std::move(write_sources))
+            , m_read_configs(std::move(read_configs))
+            , m_write_configs(std::move(write_configs))
+            , m_interactive(interactive)
+        {
+        }
+        void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) override
+        {
+            if (m_read_sources.empty() && m_read_configs.empty()) return;
+            if (plan.install_actions.empty()) return;
+
+            auto& fs = paths.get_filesystem();
+
+            std::vector<std::pair<PackageSpec, NugetReference>> nuget_refs;
+
+            for (auto&& action : plan.install_actions)
+            {
+                auto& spec = action.spec;
+                fs.remove_all_inside(paths.package_dir(spec), VCPKG_LINE_INFO);
+
+                nuget_refs.emplace_back(spec, NugetReference(action));
+            }
+
+            System::print2("Attempting to fetch ", plan.install_actions.size(), " packages from nuget.\n");
+
+            auto packages_config = paths.buildtrees / fs::u8path("packages.config");
+
+            auto generate_packages_config = [&] {
+                XmlSerializer xml;
+                xml.emit_declaration().line_break();
+                xml.open_tag("packages").line_break();
+
+                for (auto&& nuget_ref : nuget_refs)
+                    xml.start_complex_open_tag("package")
+                        .text_attr("id", nuget_ref.second.id)
+                        .text_attr("version", nuget_ref.second.version)
+                        .finish_self_closing_complex_tag()
+                        .line_break();
+
+                xml.close_tag("packages").line_break();
+                paths.get_filesystem().write_contents(packages_config, xml.buf, VCPKG_LINE_INFO);
+            };
+
+            const auto& nuget_exe = paths.get_tool_exe("nuget");
+            std::vector<std::string> cmdlines;
+
+            if (!m_read_sources.empty())
+            {
+                // First check using all sources
+                System::CmdLineBuilder cmdline;
+                cmdline.path_arg(nuget_exe)
+                    .string_arg("install")
+                    .path_arg(packages_config)
+                    .string_arg("-OutputDirectory")
+                    .path_arg(paths.packages)
+                    .string_arg("-Source")
+                    .string_arg(Strings::join(";", m_read_sources))
+                    .string_arg("-ExcludeVersion")
+                    .string_arg("-NoCache")
+                    .string_arg("-PreRelease")
+                    .string_arg("-DirectDownload")
+                    .string_arg("-PackageSaveMode")
+                    .string_arg("nupkg")
+                    .string_arg("-Verbosity")
+                    .string_arg("detailed")
+                    .string_arg("-ForceEnglishOutput");
+                if (!m_interactive) cmdline.string_arg("-NonInteractive");
+                cmdlines.push_back(cmdline.extract());
+            }
+            for (auto&& cfg : m_read_configs)
+            {
+                // Then check using each config
+                System::CmdLineBuilder cmdline;
+                cmdline.path_arg(nuget_exe)
+                    .string_arg("install")
+                    .path_arg(packages_config)
+                    .string_arg("-OutputDirectory")
+                    .path_arg(paths.packages)
+                    .string_arg("-ConfigFile")
+                    .path_arg(cfg)
+                    .string_arg("-ExcludeVersion")
+                    .string_arg("-NoCache")
+                    .string_arg("-PreRelease")
+                    .string_arg("-DirectDownload")
+                    .string_arg("-PackageSaveMode")
+                    .string_arg("nupkg")
+                    .string_arg("-Verbosity")
+                    .string_arg("detailed")
+                    .string_arg("-ForceEnglishOutput");
+                if (!m_interactive) cmdline.string_arg("-NonInteractive");
+                cmdlines.push_back(cmdline.extract());
+            }
+
+            size_t num_restored = 0;
+
+            for (const auto& cmdline : cmdlines)
+            {
+                if (nuget_refs.empty()) break;
+
+                [&] {
+                    generate_packages_config();
+                    if (Debug::g_debugging)
+                        System::cmd_execute(cmdline);
+                    else
+                    {
+                        auto res = System::cmd_execute_and_capture_output(cmdline);
+                        if (res.output.find("Authentication may require manual action.") != std::string::npos)
+                        {
+                            System::print2(
+                                System::Color::warning,
+                                "One or more NuGet credential providers requested manual action. Add the binary "
+                                "source 'interactive' to allow interactivity.\n");
+                        }
+                    }
+                }();
+
+                Util::erase_remove_if(nuget_refs, [&](const std::pair<PackageSpec, NugetReference>& nuget_ref) -> bool {
+                    auto nupkg_path = paths.package_dir(nuget_ref.first) / fs::u8path(nuget_ref.second.id + ".nupkg");
+                    if (fs.exists(nupkg_path, ignore_errors))
+                    {
+                        fs.remove(nupkg_path, VCPKG_LINE_INFO);
+                        Checks::check_exit(VCPKG_LINE_INFO,
+                                           !fs.exists(nupkg_path, ignore_errors),
+                                           "Unable to remove nupkg after restoring: %s",
+                                           nupkg_path.u8string());
+                        m_restored.emplace(nuget_ref.first);
+                        ++num_restored;
+                        return true;
+                    }
+                    else
+                    {
+                        return false;
+                    }
+                });
+            }
+
+            System::print2("Restored ", num_restored, " packages. Use --debug for more information.\n");
+        }
+        RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction& action) override
+        {
+            if (Util::Sets::contains(m_restored, action.spec))
+                return RestoreResult::success;
+            else
+                return RestoreResult::missing;
+        }
+        void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
+        {
+            if (m_write_sources.empty() && m_write_configs.empty()) return;
+            auto& spec = action.spec;
+
+            NugetReference nuget_ref(action);
+            auto nuspec_path = paths.buildtrees / spec.name() / (spec.triplet().to_string() + ".nuspec");
+            paths.get_filesystem().write_contents(
+                nuspec_path, generate_nuspec(paths, action, nuget_ref), VCPKG_LINE_INFO);
+
+            const auto& nuget_exe = paths.get_tool_exe("nuget");
+            System::CmdLineBuilder cmdline;
+            cmdline.path_arg(nuget_exe)
+                .string_arg("pack")
+                .path_arg(nuspec_path)
+                .string_arg("-OutputDirectory")
+                .path_arg(paths.buildtrees)
+                .string_arg("-NoDefaultExcludes")
+                .string_arg("-ForceEnglishOutput");
+            if (!m_interactive) cmdline.string_arg("-NonInteractive");
+
+            auto pack_rc = [&] {
+                if (Debug::g_debugging)
+                    return System::cmd_execute(cmdline);
+                else
+                    return System::cmd_execute_and_capture_output(cmdline).exit_code;
+            }();
+
+            if (pack_rc != 0)
+            {
+                System::print2(System::Color::error, "Packing NuGet failed. Use --debug for more information.\n");
+            }
+            else
+            {
+                auto nupkg_path = paths.buildtrees / nuget_ref.nupkg_filename();
+                for (auto&& write_src : m_write_sources)
+                {
+                    System::CmdLineBuilder cmd;
+                    cmd.path_arg(nuget_exe)
+                        .string_arg("push")
+                        .path_arg(nupkg_path)
+                        .string_arg("-ApiKey")
+                        .string_arg("AzureDevOps")
+                        .string_arg("-ForceEnglishOutput")
+                        .string_arg("-Source")
+                        .string_arg(write_src);
+                    if (!m_interactive) cmd.string_arg("-NonInteractive");
+
+                    System::print2("Uploading binaries for ", spec, " to NuGet source ", write_src, ".\n");
+
+                    auto rc = [&] {
+                        if (Debug::g_debugging)
+                            return System::cmd_execute(cmd);
+                        else
+                            return System::cmd_execute_and_capture_output(cmd).exit_code;
+                    }();
+
+                    if (rc != 0)
+                    {
+                        System::print2(System::Color::error,
+                                       "Pushing NuGet to ",
+                                       write_src,
+                                       " failed. Use --debug for more information.\n");
+                    }
+                }
+                for (auto&& write_cfg : m_write_configs)
+                {
+                    System::CmdLineBuilder cmd;
+                    cmd.path_arg(nuget_exe)
+                        .string_arg("push")
+                        .path_arg(nupkg_path)
+                        .string_arg("-ApiKey")
+                        .string_arg("AzureDevOps")
+                        .string_arg("-ForceEnglishOutput")
+                        .string_arg("-ConfigFile")
+                        .path_arg(write_cfg);
+                    if (!m_interactive) cmd.string_arg("-NonInteractive");
+
+                    System::print2(
+                        "Uploading binaries for ", spec, " using NuGet config ", write_cfg.u8string(), ".\n");
+
+                    auto rc = [&] {
+                        if (Debug::g_debugging)
+                            return System::cmd_execute(cmd);
+                        else
+                            return System::cmd_execute_and_capture_output(cmd).exit_code;
+                    }();
+
+                    if (rc != 0)
+                    {
+                        System::print2(System::Color::error,
+                                       "Pushing NuGet with ",
+                                       write_cfg.u8string(),
+                                       " failed. Use --debug for more information.\n");
+                    }
+                }
+                paths.get_filesystem().remove(nupkg_path, ignore_errors);
+            }
+        }
+        void push_failure(const VcpkgPaths&, const std::string&, const PackageSpec&) override {}
+        RestoreResult precheck(const VcpkgPaths&, const Dependencies::InstallPlanAction&, bool) override
+        {
+            return RestoreResult::missing;
+        }
+
+    private:
+        std::vector<std::string> m_read_sources;
+        std::vector<std::string> m_write_sources;
+
+        std::vector<fs::path> m_read_configs;
+        std::vector<fs::path> m_write_configs;
+
+        std::set<PackageSpec> m_restored;
+        bool m_interactive;
+    };
+
+    struct MergeBinaryProviders : IBinaryProvider
+    {
+        explicit MergeBinaryProviders(std::vector<std::unique_ptr<IBinaryProvider>>&& providers)
+            : m_providers(std::move(providers))
+        {
+        }
+
+        void prefetch(const VcpkgPaths& paths, const Dependencies::ActionPlan& plan) override
+        {
+            for (auto&& provider : m_providers)
+            {
+                provider->prefetch(paths, plan);
+            }
+        }
+        RestoreResult try_restore(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
+        {
+            for (auto&& provider : m_providers)
+            {
+                auto result = provider->try_restore(paths, action);
+                switch (result)
+                {
+                    case RestoreResult::build_failed:
+                    case RestoreResult::success: return result;
+                    case RestoreResult::missing: continue;
+                    default: Checks::unreachable(VCPKG_LINE_INFO);
+                }
+            }
+            return RestoreResult::missing;
+        }
+        void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) override
+        {
+            for (auto&& provider : m_providers)
+            {
+                provider->push_success(paths, action);
+            }
+        }
+        void push_failure(const VcpkgPaths& paths, const std::string& abi_tag, const PackageSpec& spec) override
+        {
+            for (auto&& provider : m_providers)
+            {
+                provider->push_failure(paths, abi_tag, spec);
+            }
+        }
+        RestoreResult precheck(const VcpkgPaths& paths,
+                               const Dependencies::InstallPlanAction& action,
+                               bool purge_tombstones) override
+        {
+            for (auto&& provider : m_providers)
+            {
+                auto result = provider->precheck(paths, action, purge_tombstones);
+                switch (result)
+                {
+                    case RestoreResult::build_failed:
+                    case RestoreResult::success: return result;
+                    case RestoreResult::missing: continue;
+                    default: Checks::unreachable(VCPKG_LINE_INFO);
+                }
+            }
+            return RestoreResult::missing;
+        }
+
+    private:
+        std::vector<std::unique_ptr<IBinaryProvider>> m_providers;
+    };
+
     struct NullBinaryProvider : IBinaryProvider
     {
-        void prefetch() override {}
+        void prefetch(const VcpkgPaths&, const Dependencies::ActionPlan&) override {}
         RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction&) override
         {
             return RestoreResult::missing;
@@ -263,6 +614,89 @@ namespace
     };
 }
 
+XmlSerializer& XmlSerializer::emit_declaration()
+{
+    buf.append(R"(<?xml version="1.0" encoding="utf-8"?>)");
+    return *this;
+}
+XmlSerializer& XmlSerializer::open_tag(StringLiteral sl)
+{
+    Strings::append(buf, '<', sl, '>');
+    indent += 2;
+    return *this;
+}
+XmlSerializer& XmlSerializer::start_complex_open_tag(StringLiteral sl)
+{
+    Strings::append(buf, '<', sl);
+    indent += 2;
+    return *this;
+}
+XmlSerializer& XmlSerializer::text_attr(StringLiteral name, StringView content)
+{
+    Strings::append(buf, ' ', name, "=\"");
+    text(content);
+    Strings::append(buf, '"');
+    return *this;
+}
+XmlSerializer& XmlSerializer::finish_complex_open_tag()
+{
+    Strings::append(buf, '>');
+    return *this;
+}
+XmlSerializer& XmlSerializer::finish_self_closing_complex_tag()
+{
+    Strings::append(buf, "/>");
+    indent -= 2;
+    return *this;
+}
+XmlSerializer& XmlSerializer::close_tag(StringLiteral sl)
+{
+    Strings::append(buf, "</", sl, '>');
+    indent -= 2;
+    return *this;
+}
+XmlSerializer& XmlSerializer::text(StringView sv)
+{
+    for (auto ch : sv)
+    {
+        if (ch == '&')
+        {
+            buf.append("&amp;");
+        }
+        else if (ch == '<')
+        {
+            buf.append("&lt;");
+        }
+        else if (ch == '>')
+        {
+            buf.append("&gt;");
+        }
+        else if (ch == '"')
+        {
+            buf.append("&quot;");
+        }
+        else if (ch == '\'')
+        {
+            buf.append("&apos;");
+        }
+        else
+        {
+            buf.push_back(ch);
+        }
+    }
+    return *this;
+}
+XmlSerializer& XmlSerializer::simple_tag(StringLiteral tag, StringView content)
+{
+    return open_tag(tag).text(content).close_tag(tag);
+}
+XmlSerializer& XmlSerializer::line_break()
+{
+    buf.push_back('\n');
+    buf.append(indent, ' ');
+    return *this;
+}
+
 IBinaryProvider& vcpkg::null_binary_provider()
 {
     static NullBinaryProvider p;
@@ -287,13 +721,42 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
 ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_configs_pure(
     const std::string& env_string, View<std::string> args)
 {
-    struct BinaryConfigParser : Parse::ParserBase
+    struct State
     {
-        using Parse::ParserBase::ParserBase;
+        bool m_cleared = false;
+        bool interactive = false;
 
         std::vector<fs::path> archives_to_read;
         std::vector<fs::path> archives_to_write;
 
+        std::vector<std::string> sources_to_read;
+        std::vector<std::string> sources_to_write;
+
+        std::vector<fs::path> configs_to_read;
+        std::vector<fs::path> configs_to_write;
+
+        void clear()
+        {
+            m_cleared = true;
+            interactive = false;
+            archives_to_read.clear();
+            archives_to_write.clear();
+            sources_to_read.clear();
+            sources_to_write.clear();
+            configs_to_read.clear();
+            configs_to_write.clear();
+        }
+    };
+
+    struct BinaryConfigParser : Parse::ParserBase
+    {
+        BinaryConfigParser(StringView text, StringView origin, State* state)
+            : Parse::ParserBase(text, origin), state(state)
+        {
+        }
+
+        State* state;
+
         void parse()
         {
             while (!at_eof())
@@ -352,8 +815,7 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
                 if (segments.size() != 1)
                     return add_error("unexpected arguments: binary config 'clear' does not take arguments",
                                      segments[1].first);
-                archives_to_read.clear();
-                archives_to_write.clear();
+                state->clear();
             }
             else if (segments[0].second == "files")
             {
@@ -381,10 +843,80 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
                     }
                     else
                     {
-                        archives_to_write.push_back(p);
+                        state->archives_to_write.push_back(p);
                     }
                 }
-                archives_to_read.push_back(std::move(p));
+                state->archives_to_read.push_back(std::move(p));
+            }
+            else if (segments[0].second == "interactive")
+            {
+                if (segments.size() > 1)
+                    return add_error("unexpected arguments: binary config 'interactive' does not accept any arguments",
+                                     segments[1].first);
+                state->interactive = true;
+            }
+            else if (segments[0].second == "nugetconfig")
+            {
+                if (segments.size() < 2)
+                    return add_error(
+                        "expected arguments: binary config 'nugetconfig' requires at least a source argument",
+                        segments[0].first);
+
+                auto p = fs::u8path(segments[1].second);
+                if (!p.is_absolute())
+                    return add_error("expected arguments: path arguments for binary config strings must be absolute",
+                                     segments[1].first);
+
+                if (segments.size() > 3)
+                {
+                    return add_error(
+                        "unexpected arguments: binary config 'nugetconfig' does not take more than 2 arguments",
+                        segments[3].first);
+                }
+                else if (segments.size() == 3)
+                {
+                    if (segments[2].second != "upload")
+                    {
+                        return add_error(
+                            "unexpected arguments: binary config 'nugetconfig' can only accept 'upload' as "
+                            "a second argument",
+                            segments[2].first);
+                    }
+                    else
+                    {
+                        state->configs_to_write.push_back(p);
+                    }
+                }
+                state->configs_to_read.push_back(std::move(p));
+            }
+            else if (segments[0].second == "nuget")
+            {
+                if (segments.size() < 2)
+                    return add_error("expected arguments: binary config 'nuget' requires at least a source argument",
+                                     segments[0].first);
+
+                auto&& p = segments[1].second;
+                if (p.empty())
+                    return add_error("unexpected arguments: binary config 'nuget' requires non-empty source");
+                if (segments.size() > 3)
+                {
+                    return add_error("unexpected arguments: binary config 'nuget' does not take more than 2 arguments",
+                                     segments[3].first);
+                }
+                else if (segments.size() == 3)
+                {
+                    if (segments[2].second != "upload")
+                    {
+                        return add_error("unexpected arguments: binary config 'nuget' can only accept 'upload' as "
+                                         "a second argument",
+                                         segments[2].first);
+                    }
+                    else
+                    {
+                        state->sources_to_write.push_back(p);
+                    }
+                }
+                state->sources_to_read.push_back(std::move(p));
             }
             else if (segments[0].second == "default")
             {
@@ -409,30 +941,152 @@ ExpectedS<std::unique_ptr<IBinaryProvider>> vcpkg::create_binary_provider_from_c
                     }
                     else
                     {
-                        archives_to_write.push_back(p);
+                        state->archives_to_write.push_back(p);
                     }
                 }
-                archives_to_read.push_back(std::move(p));
+                state->archives_to_read.push_back(std::move(p));
             }
             else
             {
-                return add_error("unknown binary provider type: valid providers are 'clear', 'default', and 'files'",
-                                 segments[0].first);
+                return add_error(
+                    "unknown binary provider type: valid providers are 'clear', 'default', 'nuget', 'nugetconfig', "
+                    "'interactive', and 'files'",
+                    segments[0].first);
             }
         }
     };
 
-    BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES");
+    State s;
+
+    BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES", &s);
     env_parser.parse();
     if (auto err = env_parser.get_error()) return err->format();
     for (auto&& arg : args)
     {
-        BinaryConfigParser arg_parser(arg, "<command>");
+        BinaryConfigParser arg_parser(arg, "<command>", &s);
         arg_parser.parse();
         if (auto err = arg_parser.get_error()) return err->format();
-        Util::Vectors::append(&env_parser.archives_to_read, arg_parser.archives_to_read);
-        Util::Vectors::append(&env_parser.archives_to_write, arg_parser.archives_to_write);
     }
-    return {std::make_unique<ArchivesBinaryProvider>(std::move(env_parser.archives_to_read),
-                                                     std::move(env_parser.archives_to_write))};
+
+    std::vector<std::unique_ptr<IBinaryProvider>> providers;
+    if (!s.archives_to_read.empty() || !s.archives_to_write.empty())
+        providers.push_back(
+            std::make_unique<ArchivesBinaryProvider>(std::move(s.archives_to_read), std::move(s.archives_to_write)));
+    if (!s.sources_to_read.empty() || !s.sources_to_write.empty() || !s.configs_to_read.empty() ||
+        !s.configs_to_write.empty())
+        providers.push_back(std::make_unique<NugetBinaryProvider>(std::move(s.sources_to_read),
+                                                                  std::move(s.sources_to_write),
+                                                                  std::move(s.configs_to_read),
+                                                                  std::move(s.configs_to_write),
+                                                                  s.interactive));
+
+    return {std::make_unique<MergeBinaryProviders>(std::move(providers))};
+}
+
+std::string vcpkg::reformat_version(const std::string& version, const std::string& abi_tag)
+{
+    static const std::regex semver_matcher(R"(v?(\d+)(\.\d+|$)(\.\d+)?.*)");
+
+    std::smatch sm;
+    if (std::regex_match(version.cbegin(), version.cend(), sm, semver_matcher))
+    {
+        auto major = trim_leading_zeroes(sm.str(1));
+        auto minor = sm.size() > 2 && !sm.str(2).empty() ? trim_leading_zeroes(sm.str(2).substr(1)) : "0";
+        auto patch = sm.size() > 3 && !sm.str(3).empty() ? trim_leading_zeroes(sm.str(3).substr(1)) : "0";
+        return Strings::concat(major, '.', minor, '.', patch, "-", abi_tag);
+    }
+
+    static const std::regex date_matcher(R"((\d\d\d\d)-(\d\d)-(\d\d).*)");
+    if (std::regex_match(version.cbegin(), version.cend(), sm, date_matcher))
+    {
+        return Strings::concat(trim_leading_zeroes(sm.str(1)),
+                               '.',
+                               trim_leading_zeroes(sm.str(2)),
+                               '.',
+                               trim_leading_zeroes(sm.str(3)),
+                               "-",
+                               abi_tag);
+    }
+
+    return Strings::concat("0.0.0-", abi_tag);
+}
+
+std::string vcpkg::generate_nuspec(const VcpkgPaths& paths,
+                                   const Dependencies::InstallPlanAction& action,
+                                   const vcpkg::NugetReference& ref)
+{
+    auto& spec = action.spec;
+    auto& scf = *action.source_control_file_location.value_or_exit(VCPKG_LINE_INFO).source_control_file;
+    auto& version = scf.core_paragraph->version;
+    std::string description =
+        Strings::concat("NOT FOR DIRECT USE. Automatically generated cache package.\n\n",
+                        scf.core_paragraph->description,
+                        "\n\nVersion: ",
+                        version,
+                        "\nTriplet/Compiler hash: ",
+                        action.abi_info.value_or_exit(VCPKG_LINE_INFO).triplet_abi.value_or_exit(VCPKG_LINE_INFO),
+                        "\nFeatures:",
+                        Strings::join(",", action.feature_list, [](const std::string& s) { return " " + s; }),
+                        "\nDependencies:\n");
+
+    for (auto&& dep : action.package_dependencies)
+    {
+        Strings::append(description, "    ", dep.name(), '\n');
+    }
+    XmlSerializer xml;
+    xml.open_tag("package").line_break();
+    xml.open_tag("metadata").line_break();
+    xml.simple_tag("id", ref.id).line_break();
+    xml.simple_tag("version", ref.version).line_break();
+    if (!scf.core_paragraph->homepage.empty()) xml.simple_tag("projectUrl", scf.core_paragraph->homepage);
+    xml.simple_tag("authors", "vcpkg").line_break();
+    xml.simple_tag("description", description).line_break();
+    xml.open_tag("packageTypes");
+    xml.start_complex_open_tag("packageType").text_attr("name", "vcpkg").finish_self_closing_complex_tag();
+    xml.close_tag("packageTypes").line_break();
+    xml.close_tag("metadata").line_break();
+    xml.open_tag("files");
+    xml.start_complex_open_tag("file")
+        .text_attr("src", (paths.package_dir(spec) / fs::u8path("**")).u8string())
+        .text_attr("target", "")
+        .finish_self_closing_complex_tag();
+    xml.close_tag("files").line_break();
+    xml.close_tag("package").line_break();
+    return std::move(xml.buf);
+}
+
+void vcpkg::help_topic_binary_caching(const VcpkgPaths&)
+{
+    System::print2(
+        System::Color::warning,
+        "** The following help documentation covers an experimental feature that will change at any time **\n\n");
+
+    HelpTableFormatter tbl;
+    tbl.text(
+        "Vcpkg can cache compiled packages to accelerate restoration on a single machine or across the network."
+        " This functionality is currently disabled by default and must be enabled by either passing `--binarycaching` "
+        "to every vcpkg command line or setting the environment variable `VCPKG_FEATURE_FLAGS` to `binarycaching`.");
+    tbl.blank();
+    tbl.blank();
+    tbl.text(
+        "Once caching is enabled, it can be further configured by either passing `--x-binarysource=<source>` options "
+        "to every command line or setting the `VCPKG_BINARY_SOURCES` environment variable to a set of sources (Ex: "
+        "\"<source>;<source>;...\"). Command line sources are interpreted after environment sources.");
+    tbl.blank();
+    tbl.blank();
+    tbl.header("Valid source strings");
+    tbl.format("clear", "Removes all previous sources");
+    tbl.format("default[,upload]", "Adds the default file-based source location (~/.vcpkg/archives).");
+    tbl.format("files,<path>[,upload]", "Adds a custom file-based source location.");
+    tbl.format("nuget,<uri>[,upload]",
+               "Adds a NuGet-based source; equivalent to the `-Source` parameter of the NuGet CLI.");
+    tbl.format("nugetconfig,<path>[,upload]",
+               "Adds a NuGet-config-file-based source; equivalent to the `-Config` parameter of the NuGet CLI. This "
+               "config should specify `defaultPushSource` for uploads.");
+    tbl.format("interactive", "Enables interactive credential management for some source types");
+    tbl.blank();
+    tbl.text("The `upload` optional parameter for certain source strings controls whether on-demand builds will be "
+             "uploaded to that remote.");
+
+    System::print2(tbl.m_str);
 }

toolsrc/src/vcpkg/build.cpp

@@ -280,7 +280,8 @@ namespace vcpkg::Build
     const System::Environment& EnvCache::get_action_env(const VcpkgPaths& paths, const AbiInfo& abi_info)
     {
 #if defined(_WIN32)
-        std::string build_env_cmd = make_build_env_cmd(*abi_info.pre_build_info, *abi_info.toolset);
+        std::string build_env_cmd =
+            make_build_env_cmd(*abi_info.pre_build_info, abi_info.toolset.value_or_exit(VCPKG_LINE_INFO));
 
         const auto& base_env = envs.get_lazy(abi_info.pre_build_info->passthrough_env_vars, [&]() -> EnvMapEntry {
             std::unordered_map<std::string, std::string> env;
@@ -324,6 +325,7 @@ namespace vcpkg::Build
     const std::string& EnvCache::get_triplet_info(const VcpkgPaths& paths, const AbiInfo& abi_info)
     {
         const auto& fs = paths.get_filesystem();
+        Checks::check_exit(VCPKG_LINE_INFO, abi_info.pre_build_info != nullptr);
         const fs::path triplet_file_path = paths.get_triplet_file_path(abi_info.pre_build_info->triplet);
 
         auto tcfile = abi_info.pre_build_info->toolchain_file();
@@ -446,7 +448,7 @@ namespace vcpkg::Build
             {"CURRENT_BUILDTREES_DIR", buildpath},
             {"CURRENT_PACKAGES_DIR", paths.packages / ("detect_compiler_" + triplet.canonical_name())},
         };
-        get_generic_cmake_build_args(paths, triplet, *abi_info.toolset, cmake_args);
+        get_generic_cmake_build_args(paths, triplet, abi_info.toolset.value_or_exit(VCPKG_LINE_INFO), cmake_args);
 
         auto command = vcpkg::make_cmake_cmd(paths, paths.ports_cmake, std::move(cmake_args));
 
@@ -517,7 +519,10 @@ namespace vcpkg::Build
             {"ALL_FEATURES", all_features},
         };
         get_generic_cmake_build_args(
-            paths, triplet, *action.abi_info.value_or_exit(VCPKG_LINE_INFO).toolset, variables);
+            paths,
+            triplet,
+            action.abi_info.value_or_exit(VCPKG_LINE_INFO).toolset.value_or_exit(VCPKG_LINE_INFO),
+            variables);
 
         if (Util::Enum::to_bool(action.build_options.only_downloads))
         {
@@ -738,11 +743,9 @@ namespace vcpkg::Build
         return result;
     }
 
-    static void abi_entries_from_abi_info(const VcpkgPaths& paths,
-                                          const AbiInfo& abi_info,
-                                          std::vector<AbiEntry>& abi_tag_entries)
+    static void abi_entries_from_abi_info(const AbiInfo& abi_info, std::vector<AbiEntry>& abi_tag_entries)
     {
-        abi_tag_entries.emplace_back("triplet", paths.get_triplet_info(abi_info));
+        abi_tag_entries.emplace_back("triplet", abi_info.triplet_abi.value_or_exit(VCPKG_LINE_INFO));
 
         const auto& pre_build_info = *abi_info.pre_build_info;
         if (pre_build_info.public_abi_override)
@@ -771,7 +774,7 @@ namespace vcpkg::Build
 
         std::vector<AbiEntry> abi_tag_entries(dependency_abis.begin(), dependency_abis.end());
 
-        abi_entries_from_abi_info(paths, action.abi_info.value_or_exit(VCPKG_LINE_INFO), abi_tag_entries);
+        abi_entries_from_abi_info(action.abi_info.value_or_exit(VCPKG_LINE_INFO), abi_tag_entries);
 
         // If there is an unusually large number of files in the port then
         // something suspicious is going on.  Rather than hash all of them
@@ -833,8 +836,7 @@ namespace vcpkg::Build
             System::print2(message);
         }
 
-        auto abi_tag_entries_missing = abi_tag_entries;
-        Util::erase_remove_if(abi_tag_entries_missing, [](const AbiEntry& p) { return !p.value.empty(); });
+        auto abi_tag_entries_missing = Util::filter(abi_tag_entries, [](const AbiEntry& p) { return p.value.empty(); });
 
         if (abi_tag_entries_missing.empty())
         {
@@ -899,7 +901,8 @@ namespace vcpkg::Build
 
             abi_info.pre_build_info = std::make_unique<PreBuildInfo>(
                 paths, action.spec.triplet(), var_provider.get_tag_vars(action.spec).value_or_exit(VCPKG_LINE_INFO));
-            abi_info.toolset = &paths.get_toolset(*abi_info.pre_build_info);
+            abi_info.toolset = paths.get_toolset(*abi_info.pre_build_info);
+            abi_info.triplet_abi = paths.get_triplet_info(abi_info);
 
             auto maybe_abi_tag_and_file = compute_abi_tag(paths, action, dependency_abis);
             if (auto p = maybe_abi_tag_and_file.get())

toolsrc/src/vcpkg/help.cpp

@@ -1,6 +1,7 @@
 #include "pch.h"
 
 #include <vcpkg/base/system.print.h>
+#include <vcpkg/binarycaching.h>
 #include <vcpkg/commands.h>
 #include <vcpkg/export.h>
 #include <vcpkg/help.h>
@@ -13,7 +14,7 @@ namespace vcpkg::Help
     {
         using topic_function = void (*)(const VcpkgPaths& paths);
 
-        constexpr Topic(CStringView n, topic_function fn) : name(n), print(fn) { }
+        constexpr Topic(CStringView n, topic_function fn) : name(n), print(fn) {}
 
         CStringView name;
         topic_function print;
@@ -40,10 +41,11 @@ namespace vcpkg::Help
         nullptr,
     };
 
-    static constexpr std::array<Topic, 13> topics = {{
+    static constexpr std::array<Topic, 15> topics = {{
+        {"binarycaching", help_topic_binary_caching},
         {"create", command_topic_fn<Commands::Create::COMMAND_STRUCTURE>},
-        {"edit", command_topic_fn<Commands::Edit::COMMAND_STRUCTURE>},
         {"depend-info", command_topic_fn<Commands::DependInfo::COMMAND_STRUCTURE>},
+        {"edit", command_topic_fn<Commands::Edit::COMMAND_STRUCTURE>},
         {"env", command_topic_fn<Commands::Env::COMMAND_STRUCTURE>},
         {"export", command_topic_fn<Export::COMMAND_STRUCTURE>},
         {"help", command_topic_fn<Help::COMMAND_STRUCTURE>},
@@ -54,13 +56,12 @@ namespace vcpkg::Help
         {"remove", command_topic_fn<Remove::COMMAND_STRUCTURE>},
         {"search", command_topic_fn<Commands::Search::COMMAND_STRUCTURE>},
         {"topics", help_topics},
+        {"triplet", help_topic_valid_triplet},
     }};
 
     static void help_topics(const VcpkgPaths&)
     {
-        System::print2("Available help topics:\n"
-                       "  triplet\n"
-                       "  integrate",
+        System::print2("Available help topics:",
                        Strings::join("", topics, [](const Topic& topic) { return std::string("\n  ") + topic.name; }),
                        "\n");
     }
@@ -110,7 +111,7 @@ namespace vcpkg::Help
             Checks::exit_success(VCPKG_LINE_INFO);
         }
         const auto& topic = args.command_arguments[0];
-        if (topic == "triplet" || topic == "triplets" || topic == "triple")
+        if (topic == "triplets" || topic == "triple")
         {
             help_topic_valid_triplet(paths);
             Checks::exit_success(VCPKG_LINE_INFO);

toolsrc/src/vcpkg/install.cpp

@@ -470,6 +470,8 @@ namespace vcpkg::Install
 
         Build::compute_all_abis(paths, action_plan, var_provider, status_db);
 
+        binaryprovider.prefetch(paths, action_plan);
+
         for (auto&& action : action_plan.install_actions)
         {
             with_tracking(action.spec, [&]() {

toolsrc/src/vcpkg/vcpkgcmdarguments.cpp

@@ -628,6 +628,8 @@ namespace vcpkg
         target.append(34, ' ');
     }
 
+    static constexpr ptrdiff_t S_MAX_LINE_LENGTH = 100;
+
     void HelpTableFormatter::format(StringView col1, StringView col2)
     {
         // 2 space, 31 col1, 1 space, 65 col2 = 99
@@ -641,29 +643,8 @@ namespace vcpkg
         {
             m_str.append(32 - col1.size(), ' ');
         }
-        const char* line_start = col2.begin();
-        const char* const e = col2.end();
-        const char* best_break = std::find_if(line_start, e, [](char ch) { return ch == ' ' || ch == '\n'; });
+        text(col2, 34);
 
-        while (best_break != e)
-        {
-            const char* next_break = std::find_if(best_break + 1, e, [](char ch) { return ch == ' ' || ch == '\n'; });
-            if (next_break - line_start > 65 || *best_break == '\n')
-            {
-                m_str.append(line_start, best_break);
-                line_start = best_break + 1;
-                best_break = next_break;
-                if (line_start != e)
-                {
-                    help_table_newline_indent(m_str);
-                }
-            }
-            else
-            {
-                best_break = next_break;
-            }
-        }
-        m_str.append(line_start, best_break);
         m_str.push_back('\n');
     }
 
@@ -681,4 +662,31 @@ namespace vcpkg
     }
 
     void HelpTableFormatter::blank() { m_str.push_back('\n'); }
+
+    // Note: this formatting code does not properly handle unicode, however all of our documentation strings are English
+    // ASCII.
+    void HelpTableFormatter::text(StringView text, int indent)
+    {
+        const char* line_start = text.begin();
+        const char* const e = text.end();
+        const char* best_break = std::find_if(line_start, e, [](char ch) { return ch == ' ' || ch == '\n'; });
+
+        while (best_break != e)
+        {
+            const char* next_break = std::find_if(best_break + 1, e, [](char ch) { return ch == ' ' || ch == '\n'; });
+            if (*best_break == '\n' || next_break - line_start + indent > S_MAX_LINE_LENGTH)
+            {
+                m_str.append(line_start, best_break);
+                m_str.push_back('\n');
+                line_start = best_break + 1;
+                best_break = next_break;
+                m_str.append(indent, ' ');
+            }
+            else
+            {
+                best_break = next_break;
+            }
+        }
+        m_str.append(line_start, best_break);
+    }
 }