(#86) Improves Offline Download Process

JPRuskin · JPRuskin · commit 0011dfa4e698 · 2024-05-14T17:25:14.000+01:00
Adds dependency packages explicitly to manifest and downloads each ignoring chains. This should hopefully allow folk to internalize everything a little more reliably.

Some packages were slipping through the cracks and not being downloaded. This could be problematic, so I ~overengineered the solution~ added some pre-existing functions in.
diff --git a/OfflineInstallPreparation.ps1 b/OfflineInstallPreparation.ps1
@@ -51,7 +51,7 @@ if (-not (Test-Path $ChocoInstallScript)) {
 $Signature = Get-AuthenticodeSignature -FilePath $ChocoInstallScript
 
 if ($Signature.Status -eq 'Valid' -and $Signature.SignerCertificate.Subject -eq 'CN="Chocolatey Software, Inc.", O="Chocolatey Software, Inc.", L=Topeka, S=Kansas, C=US') {
-    if (-not (Get-Command choco.exe)) {
+    if (-not (Get-Command choco.exe -ErrorAction SilentlyContinue)) {
         if (Test-Path $PSScriptRoot\files\chocolatey.*.nupkg) {
             $env:ChocolateyDownloadUrl = (Convert-Path $PSScriptRoot\files\chocolatey.*.nupkg)[0]
         }
@@ -88,6 +88,7 @@ foreach ($Package in (Get-Content $PSScriptRoot\files\chocolatey.json | ConvertF
     $ChocoArgs = @(
         "download", "$($Package.Name)"
         "--output-directory", $PackageWorkingDirectory
+        "--ignore-dependencies"
     )
     $ChocoArgs += switch ($Package.psobject.properties.name) {
         "Version" { "--version=$($Package.Version)" }
@@ -98,10 +99,10 @@ foreach ($Package in (Get-Content $PSScriptRoot\files\chocolatey.json | ConvertF
     }
 
     try {
-        if (-not (Test-Path "$($PackageWorkingDirectory)\$($Package.Name)*.nupkg") -and -not (Test-Path "$PSScriptRoot\files\$($Package.Name)*.nupkg")) {
+        if (-not (Get-ChocolateyPackageMetadata -Path $PackageWorkingDirectory -Id $Package.Name) -and -not (Get-ChocolateyPackageMetadata -Path "$PSScriptRoot\files\" -Id $Package.Name)) {
             Write-Host "Downloading '$($Package.Name)'"
 
-            while ((Get-ChildItem $PackageWorkingDirectory -Filter *.nupkg).Where{$_.CreationTime -gt (Get-Date).AddMinutes(-2)}.Count -gt 5) {
+            while ((Get-ChildItem $PackageWorkingDirectory -Filter *.nupkg).Where{$_.CreationTime -gt (Get-Date).AddMinutes(-1)}.Count -gt 5) {
                 Write-Verbose "Slowing down for a minute, in order to not trigger rate-limiting..."
                 Start-Sleep -Seconds 5
             }
diff --git a/Set-SslSecurity.ps1 b/Set-SslSecurity.ps1
@@ -211,6 +211,9 @@ process {
     $chocoArgs = @('apikey', "--source='$RepositoryUrl'", "--api-key='$NuGetApiKey'")
     & choco @chocoArgs
 
+    # Reset the NuGet v3 cache, such that it doesn't capture localhost as the FQDN
+    Remove-NexusRepositoryFolder -RepositoryName ChocolateyInternal -Name v3
+
     Update-JsonFile -Path "$env:SystemDrive\choco-setup\logs\nexus.json" -Properties @{
         NexusUri = "https://$($SubjectWithoutCn):8443"
         NexusRepo = $RepositoryUrl
diff --git a/Start-C4bSetup.ps1 b/Start-C4bSetup.ps1
@@ -67,7 +67,13 @@ param(
     $Thumbprint,
 
     # If provided, shows all Chocolatey output. Otherwise, blissful quiet.
-    [switch]$ShowChocoOutput
+    [switch]$ShowChocoOutput,
+
+    # The branch or Pull Request to download the C4B setup scripts from.
+    # Defaults to main.
+    [string]
+    [Alias('PR')]
+    $Branch = $env:CHOCO_QSG_BRANCH
 )
 
 if ($host.name -ne 'ConsoleHost') {
@@ -80,11 +86,16 @@ if ($ShowChocoOutput) {
     $global:PSDefaultParameterValues["Invoke-Choco:InformationAction"] = "Continue"
 }
 
-if ($env:CHOCO_QSG_DEVELOP){
-    $QsRepo = "https://github.com/chocolatey/choco-quickstart-scripts/archive/refs/heads/develop.zip"
-}
-else {
-    $QsRepo = "https://github.com/chocolatey/choco-quickstart-scripts/archive/main.zip"
+$QsRepo = if ($Branch) {
+    if ((Invoke-RestMethod -Uri "https://api.github.com/repos/chocolatey/choco-quickstart-scripts/branches").name -contains $Branch) {
+        "https://api.github.com/repos/chocolatey/choco-quickstart-scripts/zipball/$Branch"
+    } elseif ($PullRequest = Invoke-RestMethod -Uri "https://api.github.com/repos/chocolatey/choco-quickstart-scripts/pulls/$Branch" -ErrorAction SilentlyContinue) {
+        $PullRequest.head.repo.archive_url -replace '{archive_format}', 'zipball' -replace '{/ref}', "/$($PullRequest.head.ref)"
+    } else {
+        Write-Error "'$($Branch)' is not a valid branch or pull request number. Please provide a valid branch or pull request number."
+    }
+} else {
+    "https://api.github.com/repos/chocolatey/choco-quickstart-scripts/zipball/main"
 }
 
 $DefaultEap, $ErrorActionPreference = $ErrorActionPreference, 'Stop'
@@ -104,10 +115,13 @@ try {
 
     if (-not $PSScriptRoot -or $PSScriptRoot -ne $FilesDir) {
         # Download and extract C4B setup files from repo
-        Invoke-WebRequest -Uri $QsRepo -UseBasicParsing -OutFile "$TempDir\main.zip"
-        Expand-Archive "$TempDir\main.zip" $TempDir
-        Copy-Item "$TempDir\choco-quickstart-scripts-main\*" $FilesDir -Recurse
-        Remove-Item "$TempDir" -Recurse -Force
+        try {
+            Invoke-WebRequest -Uri $QsRepo -UseBasicParsing -OutFile "$TempDir\choco-quickstart-scripts.zip"
+            Expand-Archive "$TempDir\choco-quickstart-scripts.zip" $TempDir
+            Copy-Item "$TempDir\*\*" $FilesDir -Recurse -Force
+        } finally {
+            Remove-Item "$TempDir" -Recurse -Force
+        }
     }
 
     # Import Helper Functions
@@ -117,10 +131,10 @@ try {
     Write-Host "Downloading missing nupkg files to $($PkgsDir)." -ForegroundColor Green
     Write-Host "This will take some time. Feel free to get a tea or coffee." -ForegroundColor Green
 
-    & $PSScriptRoot\OfflineInstallPreparation.ps1 -LicensePath $LicenseFile
+    & $FilesDir\OfflineInstallPreparation.ps1 -LicensePath $LicenseFile
 
-    if (Test-Path $PSScriptRoot\files\*.nupkg) {
-        choco source add --name ChocolateySetup --source $PSScriptRoot\files\ --Priority 1
+    if (Test-Path $FilesDir\files\*.nupkg) {
+        choco source add --name LocalChocolateySetup --source $FilesDir\files\ --Priority 1
     }
 
     # Set Choco Server Chocolatey Configuration
diff --git a/files/chocolatey.json b/files/chocolatey.json
@@ -1,24 +1,32 @@
 {
     "packages": [
-        { "name": "KB2919442", "version": "1.0.20160915", "internalize": false },
-        { "name": "KB2919355", "version": "1.0.20160915", "internalize": false },
-        { "name": "chocolatey" },
         { "name": "chocolatey-agent" },
-        { "name": "chocolatey.extension" },
-        { "name": "chocolateygui" },
-        { "name": "chocolateygui.extension" },
-        { "name": "dotnet-aspnetcoremodule-v2", "version": "16.0.24081" },
-        { "name": "dotnet-6.0-runtime", "version": "6.0.29" },
-        { "name": "dotnet-6.0-aspnetruntime", "version": "6.0.29" },
+        { "name": "chocolatey-compatibility.extension" },
+        { "name": "chocolatey-core.extension" },
+        { "name": "chocolatey-dotnetfx.extension" },
         { "name": "chocolatey-management-database", "internalize": false },
         { "name": "chocolatey-management-service", "internalize": false },
         { "name": "chocolatey-management-web", "internalize": false },
-        { "name": "chocolatey-licensed-jenkins-scripts" },
-        { "name": "nexus-repository" },
+        { "name": "chocolatey-windowsupdate.extension" },
+        { "name": "chocolatey.extension" },
+        { "name": "chocolatey" },
+        { "name": "chocolateygui.extension" },
+        { "name": "chocolateygui" },
+        { "name": "dotnet-6.0-aspnetruntime", "version": "6.0.29" },
+        { "name": "dotnet-6.0-runtime", "version": "6.0.29" },
+        { "name": "dotnet-aspnetcoremodule-v2", "version": "16.0.24081" },
+        { "name": "dotnetfx" },
         { "name": "jenkins" },
-        { "name": "temurin21jre" },
-        { "name": "nexushell" },
+        { "name": "KB2919355", "internalize": false },
+        { "name": "KB2919442", "internalize": false },
+        { "name": "KB2999226", "internalize": false },
+        { "name": "KB3033929", "internalize": false },
+        { "name": "KB3035131", "internalize": false },
+        { "name": "KB3063858", "internalize": false },
+        { "name": "microsoft-edge" },
+        { "name": "nexus-repository" },
         { "name": "sql-server-express" },
-        { "name": "microsoft-edge" }
+        { "name": "temurin21jre" },
+        { "name": "vcredist140" }
     ]
 }
diff --git a/scripts/Create-ChocoLicensePkg.ps1 b/scripts/Create-ChocoLicensePkg.ps1
@@ -13,7 +13,7 @@ param(
     # Local path used to build the license package.
     [Parameter()]
     [string]
-    $PackagesPath = "$env:SystemDrive\choco-setup\packages",
+    $PackagesPath = "$env:SystemDrive\choco-setup\files\files",
 
     # Path to the license file.
     [Parameter()]
diff --git a/scripts/Get-Helpers.ps1 b/scripts/Get-Helpers.ps1
@@ -29,6 +29,153 @@ function Invoke-Choco {
     }
 }
 
+#region Package functions (OfflineInstallPreparation.ps1)
+if (-not ("System.IO.Compression.ZipArchive" -as [type])) {
+    Add-Type -Assembly 'System.IO.Compression'
+}
+
+function Find-FileInArchive {
+    <#
+        .Synopsis
+            Finds files with a name matching a pattern in an archive.
+        .Example
+            Find-FileInArchive -Path "C:\Archive.zip" -like "tools/files/*-x86.exe"
+        .Example
+            Find-FileInArchive -Path $Nupkg -match "tools/files/dotnetcore-sdk-(?<Version>\d+\.\d+\.\d+)-win-x86\.exe(\.ignore)?"
+        .Notes
+            Please be aware that this matches against the full name of the file, not just the file name.
+            Though given that, you can easily write something to match the file name.
+    #>
+    [CmdletBinding(DefaultParameterSetName = "match")]
+    param(
+        # Path to the archive
+        [Parameter(Mandatory)]
+        [string]$Path,
+
+        # Pattern to match with regex
+        [Parameter(Mandatory, ValueFromPipeline, ParameterSetName = "match")]
+        [string]$match,
+        
+        # Pattern to match with basic globbing
+        [Parameter(Mandatory, ParameterSetName = "like")]
+        [string]$like
+    )
+    begin {
+        while (-not $Zip -and $AccessRetries++ -lt 3) {
+            try {
+                $Stream = [IO.FileStream]::new($Path, [IO.FileMode]::Open)
+                $Zip = [IO.Compression.ZipArchive]::new($Stream, [IO.Compression.ZipArchiveMode]::Read)
+            } catch [System.IO.IOException] {
+                if ($AccessRetries -ge 3) {
+                    Write-Error -Message "Accessing '$Path' failed after $AccessRetries attempts." -TargetObject $Path
+                } else {
+                    Write-Information "Could not access '$Path', retrying..."
+                    Start-Sleep -Milliseconds 500
+                }
+            }
+        }
+    }
+    process {
+        if ($Zip) {
+            # Improve "security"?
+            $WhereBlock = [ScriptBlock]::Create("`$_.FullName -$($PSCmdlet.ParameterSetName) '$(Get-Variable -Name $PSCmdlet.ParameterSetName -ValueOnly)'")
+            $Zip.Entries | Where-Object -FilterScript $WhereBlock
+        }
+    }
+    end {
+        if ($Zip) {
+            $Zip.Dispose()
+        }
+        if ($Stream) {
+            $Stream.Close()
+            $Stream.Dispose()
+        }
+    }
+}
+
+function Get-FileContentInArchive {
+    <#
+        .Synopsis
+            Returns the content of a file from within an archive
+        .Example
+            Get-FileContentInArchive -Path $ZipPath -Name "chocolateyInstall.ps1"
+        .Example
+            Get-FileContentInArchive -Zip $Zip -FullName "tools\chocolateyInstall.ps1"
+        .Example
+            Find-FileInArchive -Path $ZipPath -Like *.nuspec | Get-FileContentInArchive
+    #>
+    [CmdletBinding(DefaultParameterSetName = "PathFullName")]
+    [OutputType([string])]
+    param(
+        # Path to the archive
+        [Parameter(Mandatory, ParameterSetName = "PathFullName")]
+        [Parameter(Mandatory, ParameterSetName = "PathName")]
+        [string]$Path,
+
+        # Zip object for the archive
+        [Parameter(Mandatory, ParameterSetName = "ZipFullName", ValueFromPipelineByPropertyName)]
+        [Parameter(Mandatory, ParameterSetName = "ZipName", ValueFromPipelineByPropertyName)]
+        [Alias("Archive")]
+        [IO.Compression.ZipArchive]$Zip,
+
+        # Name of the file(s) to remove from the archive
+        [Parameter(Mandatory, ParameterSetName = "PathFullName", ValueFromPipelineByPropertyName)]
+        [Parameter(Mandatory, ParameterSetName = "ZipFullName", ValueFromPipelineByPropertyName)]
+        [string]$FullName,
+
+        # Name of the file(s) to remove from the archive
+        [Parameter(Mandatory, ParameterSetName = "PathName")]
+        [Parameter(Mandatory, ParameterSetName = "ZipName")]
+        [string]$Name
+    )
+    begin {
+        if (-not $PSCmdlet.ParameterSetName.StartsWith("Zip")) {
+            $Stream = [IO.FileStream]::new($Path, [IO.FileMode]::Open)
+            $Zip = [IO.Compression.ZipArchive]::new($Stream, [IO.Compression.ZipArchiveMode]::Read)
+        }
+    }
+    process {
+        if (-not $FullName) {
+            $MatchingEntries = $Zip.Entries | Where-Object {$_.Name -eq $Name}
+            if ($MatchingEntries.Count -ne 1) {
+                Write-Error "File '$Name' not found in archive" -ErrorAction Stop
+            }
+            $FullName = $MatchingEntries[0].FullName
+        }
+        [System.IO.StreamReader]::new(
+            $Zip.GetEntry($FullName).Open()
+        ).ReadToEnd()
+    }
+    end {
+        if (-not $PSCmdlet.ParameterSetName.StartsWith("Zip")) {
+            $Zip.Dispose()
+            $Stream.Close()
+            $Stream.Dispose()
+        }
+    }
+}
+
+function Get-ChocolateyPackageMetadata {
+    [CmdletBinding(DefaultParameterSetName='All')]
+    param(
+        # The folder or nupkg to check
+        [Parameter(Mandatory, Position=0, ValueFromPipelineByPropertyName)]
+        [string]$Path,
+
+        # If provided, filters found packages by ID
+        [Parameter(Mandatory, Position=1, ParameterSetName='Id')]
+        [SupportsWildcards()]
+        [Alias('Name')]
+        [string]$Id = '*'
+    )
+    process {
+        Get-ChildItem $Path -Filter $Id*.nupkg | ForEach-Object {
+            ([xml](Find-FileInArchive -Path $_.FullName -Like *.nuspec | Get-FileContentInArchive)).package.metadata | Where-Object Id -like $Id
+        }
+    }
+}
+#endregion
+
 #region Nexus functions (Start-C4BNexusSetup.ps1)
 function Wait-Nexus {
     [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::tls12
