[vcpkg ci] add macos scripts to vcpkg repo (#12172)

* [vcpkg ci] add macos scripts to vcpkg repo

* CR changes

* docs stuff

.gitignore

@@ -334,3 +334,4 @@ __pycache__/
 archives
 .DS_Store
 prefab/
+*.swp

scripts/azure-pipelines/osx/Install-Prerequisites.ps1

@@ -0,0 +1,80 @@
+#!pwsh
+#Requires -Version 6.0
+
+<#
+.SYNOPSIS
+Installs the set of prerequisites for the macOS CI hosts.
+
+.DESCRIPTION
+Install-Prerequisites.ps1 installs all of the necessary prerequisites
+to run the vcpkg macOS CI in a vagrant virtual machine,
+skipping all prerequisites that are already installed.
+
+.PARAMETER Force
+Don't skip the prerequisites that are already installed.
+
+.INPUTS
+None
+
+.OUTPUTS
+None
+#>
+[CmdletBinding()]
+Param(
+    [Parameter()]
+    [Switch]$Force
+)
+
+Set-StrictMode -Version 2
+
+if (-not $IsMacOS) {
+    Write-Error 'This script should only be run on a macOS host'
+    throw
+}
+
+Import-Module "$PSScriptRoot/Utilities.psm1"
+
+$Installables = Get-Content "$PSScriptRoot/configuration/installables.json" | ConvertFrom-Json
+
+$Installables.Applications | ForEach-Object {
+    if (-not (Get-CommandExists $_.TestCommand)) {
+        Write-Host "$($_.Name) not installed; installing now"
+    } elseif ($Force) {
+        Write-Host "$($_.Name) found; attempting to upgrade or re-install"
+    } else {
+        Write-Host "$($_.Name) already installed"
+        return
+    }
+
+    $pathToDmg = "~/Downloads/$($_.Name).dmg"
+    Get-RemoteFile -OutFile $pathToDmg -Uri $_.DmgUrl -Sha256 $_.Sha256
+
+    hdiutil attach $pathToDmg -mountpoint /Volumes/setup-installer
+    sudo installer -pkg "/Volumes/setup-installer/$($_.InstallerPath)" -target /
+    hdiutil detach /Volumes/setup-installer
+}
+
+# Install plugins
+$installedExtensionPacks = Get-InstalledVirtualBoxExtensionPacks
+
+$Installables.VBoxExtensions | ForEach-Object {
+    $extension = $_
+    $installedExts = $installedExtensionPacks | Where-Object { $_.Pack -eq $extension.FullName -and $_.Usable -eq 'true' }
+
+    if ($null -eq $installedExts) {
+        Write-Host "VBox extension: $($extension.Name) not installed; installing now"
+    } elseif ($Force) {
+        Write-Host "VBox extension: $($extension.Name) found; attempting to upgrade or re-install"
+    } else {
+        Write-Host "VBox extension: $($extension.Name) already installed"
+        return
+    }
+
+    $pathToExt = "~/Downloads/$($extension.FullName -replace ' ','_').vbox-extpack"
+
+    Get-RemoteFile -OutFile $pathToExt -Uri $extension.Url -Sha256 $extension.Sha256 | Out-Null
+
+    Write-Host 'Attempting to install extension with sudo; you may need to enter your password'
+    sudo VBoxManage extpack install --replace $pathToExt
+    sudo VBoxManage extpack cleanup
+}

scripts/azure-pipelines/osx/README.md

@@ -0,0 +1,69 @@
+# `vcpkg-eg-mac` VMs
+
+## Table of Contents
+
+- [`vcpkg-eg-mac` VMs](#vcpkg-eg-mac-vms)
+  - [Table of Contents](#table-of-contents)
+  - [Basic Usage](#basic-usage)
+  - [Setting up a new macOS machine](#setting-up-a-new-macos-machine)
+
+## Basic Usage
+
+The simplest usage, and one which should be used for when spinning up
+new VMs, and when restarting old ones, is a simple:
+
+```
+$ cd ~/vagrant/vcpkg-eg-mac
+$ vagrant up
+```
+
+Any modifications to the machines should be made in `configuration/VagrantFile`
+and `Setup-VagrantMachines.ps1`, and make sure to push any changes!
+
+## Setting up a new macOS machine
+
+Before anything else, one must download `brew` and `powershell`.
+
+```sh
+$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+$ brew cask install powershell
+```
+
+Then, we need to download the `vcpkg` repository:
+
+```sh
+$ git clone https://github.com/microsoft/vcpkg
+```
+
+And now all we need to do is set it up! Replace `XX` with the number of
+the virtual machine. Generally, that should be the same as the number
+for the physical machine; i.e., vcpkgmm-04 will have vcpkg-eg-mac-04.
+
+```sh
+$ cd vcpkg/scripts/azure-pipelines/osx
+$ ./Install-Prerequisites.ps1 -Force
+  # NOTE: you may get an error about CoreCLR; see the following paragraph if you do
+$ ./Setup-VagrantMachines.ps1 XX \
+  -Pat '<get this from azure>' \
+  -ArchivesUsername '<get this from the archives share>' \
+  -ArchivesAccessKey '<get this from the archives share>' \
+  -ArchivesUrn '<something>.file.core.windows.net' \
+  -ArchivesShare 'archives'
+$ cd ~/vagrant/vcpkg-eg-mac
+$ vagrant up
+```
+
+If you see the following error:
+
+```
+Failed to initialize CoreCLR, HRESULT: 0x8007001F
+```
+
+You have to reboot the machine; run
+
+```sh
+$ sudo shutdown -r now
+```
+
+and wait for the machine to start back up. Then, start again from
+`Install-Prerequisites.ps1`.

scripts/azure-pipelines/osx/Setup-VagrantMachines.ps1

@@ -0,0 +1,113 @@
+#!pwsh
+#Requires -Version 6.0
+
+<#
+.SYNOPSIS
+Sets up the configuration for the vagrant virtual machines.
+
+.DESCRIPTION
+Setup-VagrantMachines.ps1 sets up the virtual machines for
+vcpkg's macOS CI. It puts the VagrantFile and necessary
+configuration JSON file into ~/vagrant/vcpkg-eg-mac.
+
+.PARAMETER Pat
+The personal access token which has Read & Manage permissions on the ADO pool.
+
+.PARAMETER ArchivesUsername
+The username for the archives share.
+
+.PARAMETER ArchivesAccessKey
+The access key for the archives share.
+
+.PARAMETER ArchivesUri
+The URN of the archives share; looks like `foo.windows.core.net`.
+
+.PARAMETER ArchivesShare
+The archives share name.
+
+.PARAMETER BaseName
+The base name for the vagrant VM; the machine name is $BaseName-$MachineIdentifiers.
+Defaults to 'vcpkg-eg-mac'.
+
+.PARAMETER Force
+Delete any existing vagrant/vcpkg-eg-mac directory.
+
+.PARAMETER DiskSize
+The size to make the temporary disks in gigabytes. Defaults to 425.
+
+.PARAMETER MachineIdentifiers
+The numbers to give the machines; should match [0-9]{2}.
+
+.INPUTS
+None
+
+.OUTPUTS
+None
+#>
+[CmdletBinding(PositionalBinding=$False)]
+Param(
+    [Parameter(Mandatory=$True)]
+    [String]$Pat,
+
+    [Parameter(Mandatory=$True)]
+    [String]$ArchivesUsername,
+
+    [Parameter(Mandatory=$True)]
+    [String]$ArchivesAccessKey,
+
+    [Parameter(Mandatory=$True)]
+    [String]$ArchivesUrn,
+
+    [Parameter(Mandatory=$True)]
+    [String]$ArchivesShare,
+
+    [Parameter()]
+    [String]$BaseName = 'vcpkg-eg-mac',
+
+    [Parameter()]
+    [Switch]$Force,
+
+    [Parameter()]
+    [Int]$DiskSize = 425,
+
+    [Parameter(Mandatory=$True, ValueFromRemainingArguments)]
+    [String[]]$MachineIdentifiers
+)
+
+Set-StrictMode -Version 2
+
+if (-not $IsMacOS) {
+    throw 'This script should only be run on a macOS host'
+}
+
+if (Test-Path '~/vagrant') {
+    if ($Force) {
+        Write-Host 'Deleting existing directories'
+        Remove-Item -Recurse -Force -Path '~/vagrant' | Out-Null
+    } else {
+        throw '~/vagrant already exists; try re-running with -Force'
+    }
+}
+
+Write-Host 'Creating new directories'
+New-Item -ItemType 'Directory' -Path '~/vagrant' | Out-Null
+New-Item -ItemType 'Directory' -Path '~/vagrant/vcpkg-eg-mac' | Out-Null
+
+Copy-Item `
+    -Path "$PSScriptRoot/configuration/VagrantFile" `
+    -Destination '~/vagrant/vcpkg-eg-mac/VagrantFile'
+
+$configuration = @{
+    pat = $Pat;
+    base_name = $BaseName;
+    machine_identifiers = $MachineIdentifiers;
+    disk_size = $DiskSize;
+    archives = @{
+        username = $ArchivesUsername;
+        access_key = $ArchivesAccessKey;
+        url = $ArchivesUri;
+        share = $ArchivesShare;
+    };
+}
+ConvertTo-Json -InputObject $configuration -Depth 5 `
+    | Set-Content -Path '~/vagrant/vcpkg-eg-mac/vagrant-configuration.json'

scripts/azure-pipelines/osx/Utilities.psm1

@@ -0,0 +1,150 @@
+#Requires -Version 6.0
+Set-StrictMode -Version 2
+
+<#
+.SYNOPSIS
+Returns whether the specified command exists in the current environment.
+
+.DESCRIPTION
+Get-CommandExists takes a string as a parameter,
+and returns whether it exists in the current environment;
+either a function, alias, or an executable in the path.
+It's somewhat equivalent to `which`.
+
+.PARAMETER Name
+Specifies the name of the command which may or may not exist.
+
+.INPUTS
+System.String
+    The name of the command.
+
+.OUTPUTS
+System.Boolean
+    Whether the command exists.
+#>
+function Get-CommandExists
+{
+    [CmdletBinding()]
+    [OutputType([Boolean])]
+    Param(
+        [Parameter(ValueFromPipeline)]
+        [String]$Name
+    )
+
+    $null -ne (Get-Command -Name $Command -ErrorAction SilentlyContinue)
+}
+
+<#
+.SYNOPSIS
+Downloads a file and checks its hash.
+
+.DESCRIPTION
+Get-RemoteFile takes a URI and a hash,
+downloads the file at that URI to OutFile,
+and checks that the hash of the downloaded file.
+It then returns a FileInfo object corresponding to the downloaded file.
+
+.PARAMETER OutFile
+Specifies the file path to download to.
+
+.PARAMETER Uri
+The URI to download from.
+
+.PARAMETER Sha256
+The expected SHA256 of the downloaded file.
+
+.INPUTS
+None
+
+.OUTPUTS
+System.IO.FileInfo
+    The FileInfo for the downloaded file.
+#>
+function Get-RemoteFile
+{
+    [CmdletBinding(PositionalBinding=$False)]
+    [OutputType([System.IO.FileInfo])]
+    Param(
+        [Parameter(Mandatory=$True)]
+        [String]$OutFile,
+        [Parameter(Mandatory=$True)]
+        [String]$Uri,
+        [Parameter(Mandatory=$True)]
+        [String]$Sha256
+    )
+
+    Invoke-WebRequest -OutFile $OutFile -Uri $Uri
+    $actualHash = Get-FileHash -Algorithm SHA256 -Path $OutFile
+
+    if ($actualHash.Hash -ne $Sha256) {
+        throw @"
+Invalid hash for file $OutFile;
+    expected: $Hash
+    found:    $($actualHash.Hash)
+Please make sure that the hash in the powershell file is correct.
+"@
+    }
+
+    Get-Item $OutFile
+}
+
+<#
+.SYNOPSIS
+Gets the list of installed extensions as powershell objects.
+
+.DESCRIPTION
+Get-InstalledVirtualBoxExtensionPacks gets the installed extensions,
+returning objects that look like:
+
+{
+    Pack = 'Oracle VM VirtualBox Extension Pack';
+    Version = '6.1.10';
+    ...
+}
+
+.INPUTS
+None
+
+.OUTPUTS
+PSCustomObject
+    The list of VBox Extension objects that are installed.
+#>
+function Get-InstalledVirtualBoxExtensionPacks
+{
+    [CmdletBinding()]
+    [OutputType([PSCustomObject])]
+    Param()
+
+    $lines = VBoxManage list extpacks
+
+    $result = @()
+
+    $currentObject = $null
+    $currentKey = ""
+    $currentString = ""
+
+    $lines | ForEach-Object {
+        if ($Line[0] -eq ' ') {
+            $currentString += "`n$($Line.Trim())"
+        } else {
+            if ($null -ne $currentObject) {
+                $currentObject.$currentKey = $currentString
+            }
+            $currentKey, $currentString = $Line -Split ':'
+            $currentString = $currentString.Trim()
+
+            if ($currentKey.StartsWith('Pack no')) {
+                $currentKey = 'Pack'
+                if ($null -ne $currentObject) {
+                    Write-Output ([PSCustomObject]$currentObject)
+                }
+                $currentObject = @{}
+            }
+        }
+    }
+
+    if ($null -ne $currentObject) {
+        $currentObject.$currentKey = $currentString
+        Write-Output ([PSCustomObject]$currentObject)
+    }
+}

scripts/azure-pipelines/osx/configuration/VagrantFile

@@ -0,0 +1,118 @@
+require 'json'
+
+require "erb"
+include ERB::Util
+
+configuration = JSON.parse(File.read('./vagrant-configuration.json'))
+
+servers = configuration['machine_identifiers'].map do |id|
+  {
+    :hostname => "#{configuration['base_name']}-#{id}",
+    :box => 'ramsey/macos-catalina',
+    :ram => 12000,
+    :cpu => 5
+  }
+end
+
+brew_formulas = [
+  'autoconf',
+  'automake',
+  'libtool',
+  'bison' ]
+
+brew_cask_formulas = [
+  'powershell',
+  'gfortran' ]
+
+azure_agent_url = 'https://vstsagentpackage.azureedge.net/agent/2.171.1/vsts-agent-osx-x64-2.171.1.tar.gz'
+devops_url = 'https://dev.azure.com/vcpkg'
+agent_pool = 'vcpkgAgentPool'
+pat = configuration['pat']
+archives = configuration['archives']
+archives_url = "//#{archives['username']}:#{url_encode(archives['access_key'])}@#{archives['url']}/#{archives['share']}"
+
+Vagrant.configure('2') do |config|
+  # give them extra time to boot up
+  config.vm.boot_timeout = 600
+
+  servers.each do |machine|
+    config.vm.define machine[:hostname] do |node|
+
+      node.vm.box = machine[:box]
+      node.vm.hostname = machine[:hostname]
+      node.vm.synced_folder '.', '/vagrant', disabled: true
+
+      node.vm.disk :disk, name: "#{machine[:hostname]}-data", size: "#{config['disk_size']}GB"
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Format and mount the data filesystem',
+        inline: 'diskutil partitionDisk /dev/disk0 1 GPT jhfs+ data 0',
+        privileged: true
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Link the data filesystem to the home directory',
+        inline: "ln -s /Volumes/data ~/Data",
+        privileged: false
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Download azure agent',
+        inline: "curl -s -o ~/Downloads/azure-agent.tar.gz #{azure_agent_url}",
+        privileged: false
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Unpack azure agent',
+        inline: 'mkdir myagent; cd myagent; tar xf ~/Downloads/azure-agent.tar.gz',
+        privileged: false
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Install brew and xcode command line tools',
+        inline: '/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"',
+        privileged: false
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Install brew applications',
+        inline: "brew install #{brew_formulas.join(' ')} && brew cask install #{brew_cask_formulas.join(' ')}",
+        privileged: false
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Create archives mountpoint',
+        inline: 'mkdir ~/Data/archives',
+        privileged: false
+
+      node.vm.provision "shell",
+        run: 'once',
+        name: 'Mount archives directory',
+        inline: "mount_smbfs -d 777 -f 777 #{archives_url} ~/Data/archives",
+        privileged: false
+
+      node.vm.provision 'shell',
+        run: 'once',
+        name: 'Add VM to azure agent pool',
+        inline: "cd ~/myagent;\
+          ./config.sh --unattended \
+            --url #{devops_url} \
+            --work ~/Data/work \
+            --auth pat --token #{pat} \
+            --pool #{agent_pool} \
+            --agent `hostname` \
+            --replace \
+            --acceptTeeEula",
+        privileged: false
+
+      # Start listening for jobs
+      node.vm.provision 'shell',
+        run: 'always',
+        name: 'Start running azure pipelines',
+        inline: 'cd /Users/vagrant/myagent;\
+          nohup ./run.sh&',
+        privileged: false
+    end
+  end
+end

scripts/azure-pipelines/osx/configuration/installables.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "./installables.schema.json",
+
+  "Applications": [
+    {
+      "Name": "VirtualBox",
+      "TestCommand": "VBoxManage",
+      "DmgUrl": "https://download.virtualbox.org/virtualbox/6.1.10/VirtualBox-6.1.10-138449-OSX.dmg",
+      "Sha256": "EF0CA4924922514B6AD71469998821F2CF7C596B4B8B59736C3699759E0F1DF8",
+      "InstallerPath": "VirtualBox.pkg"
+    },
+    {
+      "Name": "vagrant",
+      "TestCommand": "vagrant",
+      "DmgUrl": "https://releases.hashicorp.com/vagrant/2.2.9/vagrant_2.2.9_x86_64.dmg",
+      "Sha256": "529CDE2A78E6DF38EC906B65C70B36A087E2601EAB42E25856E35B20CCB027C0",
+      "InstallerPath": "vagrant.pkg"
+    }
+  ],
+  "VBoxExtensions": [
+    {
+      "Name": "Extension Pack",
+      "FullName": "Oracle VM VirtualBox Extension Pack",
+      "Url": "https://download.virtualbox.org/virtualbox/6.1.10/Oracle_VM_VirtualBox_Extension_Pack-6.1.10.vbox-extpack",
+      "Sha256": "03067F27F4DA07C5D0FDAFC56D27E3EA23A60682B333B2A1010FB74EF9A40C28"
+    }
+  ]
+}

scripts/azure-pipelines/osx/configuration/installables.schema.json

@@ -0,0 +1,61 @@
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
+  "type": "object",
+  "definitions": {
+    "sha256": {
+      "type": "string",
+      "pattern": "[A-Z0-9]{64}"
+    }
+  },
+  "required": [
+    "Applications",
+    "VBoxExtensions"
+  ],
+  "properties": {
+    "Applications": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "Name": {
+            "type": "string"
+          },
+          "TestCommand": {
+            "type": "string"
+          },
+          "DmgUrl": {
+            "type": "string",
+            "format": "uri"
+          },
+          "Sha256": {
+            "$ref": "#/definitions/sha256"
+          },
+          "InstallerPath": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "VBoxExtensions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "Name": {
+            "type": "string"
+          },
+          "FullName": {
+            "type": "string"
+          },
+          "Url": {
+            "type": "string",
+            "format": "uri"
+          },
+          "Sha256": {
+            "$ref": "#/definitions/sha256"
+          }
+        }
+      }
+    }
+  }
+}

scripts/azure-pipelines/osx/configuration/vagrant-configuration.schema.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "https://json-schema.org/draft/2019-09/schema",
+
+  "type": "object",
+
+  "required": [
+    "pat",
+    "base_name",
+    "disk_size",
+    "machine_identifiers",
+    "archives"
+  ],
+
+  "properties": {
+    "pat": {
+      "type": "string"
+    },
+    "base_name": {
+      "type": "string"
+    },
+    "disk_size": {
+      "type": "integer"
+    },
+    "machine_identifiers": {
+      "type": "array",
+
+      "items": {
+        "type": "string",
+        "pattern": "[0-9]{2}"
+      }
+    },
+    "archives": {
+      "type": "object",
+      "required": [
+        "username",
+        "access_key",
+        "url",
+        "share"
+      ],
+      "properties": {
+        "username": { "type": "string" },
+        "access_key": { "type": "string" },
+        "url": { "type": "string" },
+        "share": { "type": "string" }
+      }
+    }
+  }
+}