#requires -version 7.2

using namespace Microsoft.PowerShell.Commands

using namespace NuGet.Versioning

using namespace System.Collections

using namespace System.Collections.Concurrent

using namespace System.Collections.Generic

using namespace System.Collections.Specialized

using namespace System.IO

using namespace System.IO.Compression

using namespace System.IO.Pipelines

using namespace System.Management.Automation

using namespace System.Management.Automation.Language

using namespace System.Net

using namespace System.Net.Http

using namespace System.Reflection

using namespace System.Runtime.Caching

using namespace System.Text

using namespace System.Threading

using namespace System.Threading.Tasks

#Because we are changing state, we want to be safe

#TODO: Implement logic to only fail on module installs, such that one module failure doesn't prevent others from installing.

#Probably need to take into account inconsistent state, such as if a dependent module fails then the depending modules should be removed.

$ErrorActionPreference = 'Stop'

if ($ENV:CI) {

Write-Verbose 'CI Environment Variable is set, this indicates a Continuous Integration System is being used. ModuleFast will suppress prompts by setting ConfirmPreference to None and forcing confirmations to false. This is to ensure that ModuleFast can be used in CI/CD systems without user interaction.'

#Module Scope which should carry to other called commands

$SCRIPT:ConfirmPreference = 'None'

$PSDefaultParameterValues['Install-ModuleFast:Confirm'] = $false

}

#Default Source is PWSH Gallery

$SCRIPT:DefaultSource = 'https://pwsh.gallery/index.json'

#region Public

enum InstallScope {

CurrentUser

}

function Install-ModuleFast {

<#

.SYNOPSIS

High performance, declarative Powershell Module Installer

.DESCRIPTION

ModuleFast is a high performance, declarative PowerShell module installer. It is optimized for speed and written primarily in PowerShell and can be bootstrapped in a single line of code. It is ideal for Continuous Integration/Deployment and serverless scenarios where you want to install modules quickly and without any user interaction. It is inspired by PNPm (https://pnpm.io/) and other high performance declarative package managers.

ModuleFast accepts a variety of familiar PowerShell syntaxes and objects for module specification as well as a custom shorthand syntax allowing complex version requirements to be defined in a single string.

ModuleFast can also install the required modules specified in the #Requires line of a script, or in the RequiredModules section of a module manifest, by simplying providing the path to that file in the -Path parameter (which also accepts remote UNC, http, and https URLs).

---------------------------

Module Specification Syntax

---------------------------

ModuleFast supports a shorthand string syntax for defining module specifications. It generally takes the form of '<Module><Operator><Version>'. The version supports SemVer 2 and prerelease tags.

The available operators are:

- '=': Exact version match. Examples: 'ImportExcel=7.1.0', 'ImportExcel=7.1.0-preview'

- '>': Greater than. Example: 'ImportExcel>7.1.0'

- '>=': Greater than or equal to. Example: 'ImportExcel>=7.1.0'

- '<': Less than. Example: 'ImportExcel<7.1.0'

- '<=': Less than or equal to. Example: 'ImportExcel<=7.1.0'

- '!': A prerelease operator that can be present at the beginning or end of a module name to indicate that prerelease versions are acceptable. Example: 'ImportExcel!', '!ImportExcel'. It can be combined with the other operators like so: 'ImportExcel!>7.1.0'

- ':': Lets you specify a NuGet version range. Example: 'ImportExcel:(7.0.0, 7.2.1-preview]'

For more information about NuGet version range syntax used with the ':' operator: https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#version-ranges. Wilcards are supported with this syntax e.g. 'ImportExcel:3.2.*' will install the latest 3.2.x version.

ModuleFast also fully supports the ModuleSpecification object and hashtable-like string syntaxes that are used by Install-Module and Install-PSResource. More information on this format: https://learn.microsoft.com/en-us/dotnet/api/microsoft.powershell.commands.modulespecification?view=powershellsdk-7.4.0

-------

Logging

-------

Modulefast has extensive Verbose and Debug information available if you specify the -Verbose and/or -Debug parameters. This can be useful for troubleshooting or understanding how ModuleFast is working. Verbose level provides a high level "what" view of the process of module selection, while Debug level provides a much more detailed "Why" information about the module selection and installation process that can be useful in troubleshooting issues.

-----------------

Installation Path

-----------------

ModuleFast will install modules to the default PowerShell module path on non-Windows platforms. On Windows, it will install to %LOCALAPPDATA%\PowerShell\Modules by default as the default Documents PowerShell Modules folder has increasing caused problems due to conflicts with document syncing programs such as OneDrive. You can override this behavior by specifying the -Destination parameter. You can also specify 'CurrentUser' to install to the legacy Documents PowerShell Modules folder on Windows only.

As part of this installation process on Windows, ModuleFast will add the destination to your PSModulePath for the current session. This is done to ensure that the modules are available for use in the current session. If you do not want this behavior, you can specify the -NoPSModulePathUpdate switch.

In addition, if you do not already have the %LOCALAPPDATA%\PowerShell\Modules in your $env:PSModulesPath, Modulefast will append a command to add it to your user profile. This is done to ensure that the modules are available for use in future sessions. If you do not want this behavior, you can specify the -NoProfileUpdate switch.

-------

Caching

-------

ModuleFast will cache the results of the module selection process in memory for the duration of the PowerShell session. This is done to improve performance when multiple modules are being installed. If you want to clear the cache, you can call Clear-ModuleFastCache.

.PARAMETER WhatIf

Outputs the installation plan of modules not already available and needing to be installed to the pipeline as well as the console. This can be saved and provided to Install-ModuleFast at a later date.

.EXAMPLE

Install-ModuleFast 'ImportExcel' -PassThru

Installs the latest version of ImportExcel and outputs info about the installed modules. Remove -PassThru for a quieter installation, or add -Verbose or -Debug for even more information.

--- RESULT ---

Name ModuleVersion

---- -------------

ImportExcel 7.8.6

.EXAMPLE

Install-ModuleFast 'ImportExcel','VMware.PowerCLI.Sdk=12.6.0.19600125','PowerConfig<0.1.6','Az.Compute:7.1.*' -WhatIf

Prepares an install plan for the latest version of ImportExcel, a specific version of VMware.PowerCLI.Sdk, a version of PowerConfig less than 0.1.6, and the latest version of Az.Compute that is in the 7.1.x range.

--- RESULT ---

Name ModuleVersion

---- -------------

VMware.PowerCLI.Sdk 12.6.0.19600125

ImportExcel 7.8.6

PowerConfig 0.1.5

Az.Compute 7.1.0

VMware.PowerCLI.Sdk.Types 12.6.0.19600125

Az.Accounts 2.13.2

.EXAMPLE

'ImportExcel','VMware.PowerCLI.Sdk=12.6.0.19600125','PowerConfig<0.1.6','Az.Compute:7.1.*' | Install-ModuleFast -WhatIf

Same as the previous example, but using the pipeline to provide the module specifications.

--- RESULT ---

Name ModuleVersion

---- -------------

VMware.PowerCLI.Sdk 12.6.0.19600125

ImportExcel 7.8.6

PowerConfig 0.1.5

Az.Compute 7.1.0

VMware.PowerCLI.Sdk.Types 12.6.0.19600125

Az.Accounts 2.13.2

.EXAMPLE

$plan = Install-ModuleFast 'ImportExcel' -WhatIf

$plan | Install-ModuleFast

Stores an Install Plan for ImportExcel in the $plan variable, then installs it. The later install can be done later, and the $plan objects are serializable to CLIXML/JSON/etc. for storage.

--- RESULT ---

What if: Performing the operation "Install 1 Modules" on target "C:\Users\JGrote\AppData\Local\powershell\Modules". Name ModuleVersion

Name ModuleVersion

---- -------------

ImportExcel 7.8.6

.EXAMPLE

@{ModuleName='ImportExcel';ModuleVersion='7.8.6'} | Install-ModuleFast

Installs a specific version of ImportExcel using

.EXAMPLE

Install-ModuleFast 'ImportExcel' -Destination 'CurrentUser'

Installs ImportExcel to the legacy PowerShell Modules folder in My Documents on Windows only, but does not update the PSModulePath or the user profile to include the new module path. This behavior is similar to Install-Module or Install-PSResource.

.EXAMPLE

Install-ModuleFast -Path 'path\to\RequiresScript.ps1' -WhatIf

Prepares a plan to install the dependencies defined in the #Requires statement of a script if a .ps1 is specified, the #Requires statement of a module if .psm1 is specified, or the RequiredModules section of a .psd1 file if it is a PowerShell Module Manifest. This is useful for quickly installing dependencies for scripts or modules.

RequiresScript.ps1 Contents:

#Requires -Module PreReleaseTest

--- RESULT ---

What if: Performing the operation "Install 1 Modules" on target "C:\Users\JGrote\AppData\Local\powershell\Modules".

Name ModuleVersion

---- -------------

PrereleaseTest 0.0.1

.EXAMPLE

Install-ModuleFast -WhatIf

If no Specification or Path parameter is provided, ModuleFast will search for a .requires.json or a .requires.psd1 file in the current directory and install the modules specified in that file. This is useful for quickly installing dependencies for scripts or modules during a CI build or Github Action.

Note that for this format you can explicitly specify 'latest' or ':*' as the version to install the latest version of a module.

Module.requires.psd1 Contents:

@{

ImportExcel = 'latest'

'VMware.PowerCLI.Sdk' = '>=12.6.0.19600125'

}

--- RESULT ---

Name ModuleVersion

---- -------------

ImportExcel 7.8.6

VMware.PowerCLI.Sdk 12.6.0.19600125

VMware.PowerCLI.Sdk.Types 12.6.0.19600125

.EXAMPLE

Install-ModuleFast 'ImportExcel' -CI #This will write a lockfile to the current directory

Install-ModuleFast -CI #This will use the previously created lockfile to install same state as above.

If the -CI switch is specified, ModuleFast will write a lockfile to the current directory indicating all modules that were installed. This lockfile will contain the exact versions of the modules that were installed. If the lockfile is present in the future, ModuleFast will only install the versions specified in the lockfile, which is useful for reproducing CI builds even if newer versions of modules are releases that match the initial specification.

For instance, the above will install the latest version of ImportExcel (7.8.6 as of this writing) but will not install newer while modulefast is in this directory until the lockfile is removed or replaced.

#>

[CmdletBinding(SupportsShouldProcess, DefaultParameterSetName = 'Specification')]

param(

#The module(s) to install. This can be a string, a ModuleSpecification, a hashtable with nuget version style (e.g. @{Name='test';Version='1.0'}), a hashtable with ModuleSpecification style (e.g. @{Name='test';RequiredVersion='1.0'}),

[Alias('Name')]

[Alias('ModuleToInstall')]

[Alias('ModulesToInstall')]

[AllowNull()]

[AllowEmptyCollection()]

[Parameter(Position = 0, ValueFromPipeline, ParameterSetName = 'Specification')][ModuleFastSpec[]]$Specification,

#Provide a required module specification path to install from. This can be a local psd1/json file, or a remote URL with a psd1/json file in supported manifest formats, or a .ps1/.psm1 file with a #Requires statement.

[Parameter(Mandatory, ParameterSetName = 'Path')][string]$Path,

#Explicitly specify the type of SpecFile to use. ModuleFast has some limited autodetection capability for ModuleBuilder and PSDepend formats, you should use this parameter if they explicitly fail. This is ignored if the file is not a .psd1 file.

[Parameter(ParameterSetName = 'Path')][SpecFileType]$SpecFileType = [SpecFileType]::AutoDetect,

#Where to install the modules. This defaults to the builtin module path on non-windows and a custom LOCALAPPDATA location on Windows. You can also specify 'CurrentUser' to install to the Documents folder on Windows Only (this is not recommended)

[string]$Destination,

#The repository to scan for modules. TODO: Multi-repo support

[string]$Source = $SCRIPT:DefaultSource,

#The credential to use to authenticate. Only basic auth is supported

[PSCredential]$Credential,

#By default will modify your PSModulePath to use the builtin destination if not present. Setting this implicitly skips profile update as well.

[Switch]$NoPSModulePathUpdate,

#Setting this won't add the default destination to your powershell.config.json. This really only matters on Windows.

[Switch]$NoProfileUpdate,

#Setting this will check for newer modules if your installed modules are not already at the upper bound of the required version range. Note that specifying this will also clear the local request cache for remote repositories which will result in slower evaluations if the information has not changed.

[Switch]$Update,

#Prerelease packages will be included in ModuleFast evaluation. If a non-prerelease package has a prerelease dependency, that dependency will be included regardless of this setting. If this setting is specified, all packages will be evaluated for prereleases regardless of if they have a prerelease indicator such as '!' in their specification name, but will still be subject to specification version constraints that would prevent a prerelease from installing.

[Switch]$Prerelease,

#Using the CI switch will write a lockfile to the current folder. If this file is present and -CI is specified in the future, ModuleFast will only install the versions specified in the lockfile, which is useful for reproducing CI builds even if newer versions of software come out.

[Switch]$CI,

#Only consider the specified destination and not any other paths currently in the PSModulePath. This is useful for CI scenarios where you want to ensure that the modules are installed in a specific location.

[Switch]$DestinationOnly,

#How many concurrent installation threads to run. Each installation thread, given sufficient bandwidth, will likely saturate a full CPU core with decompression work. This defaults to the number of logical cores on the system. If your system uses HyperThreading and presents more logical cores than physical cores available, you may want to set this to half your number of logical cores for best performance.

[int]$ThrottleLimit = [Environment]::ProcessorCount,

#The path to the lockfile. By default it is requires.lock.json in the current folder. This is ignored if -CI parameter is not present. It is generally not recommended to change this setting.

[string]$CILockFilePath = $(Join-Path $PWD 'requires.lock.json'),

#A list of ModuleFastInfo objects to install. This parameterset is used when passing a plan to ModuleFast via the pipeline and is generally not used directly.

[Parameter(Mandatory, ValueFromPipeline, ParameterSetName = 'ModuleFastInfo')][ModuleFastInfo[]]$ModuleFastInfo,

#Outputs the installation plan of modules not already available and needing to be installed to the pipeline as well as the console. This can be saved and provided to Install-ModuleFast at a later date. This is functionally the same as -WhatIf but without the additional WhatIf Output

[Switch]$Plan,

#This will output the resulting modules that were installed.

[Switch]$PassThru,

#Setting this to "CurrentUser" is the same as specifying the destination as 'Current'. This is a usability convenience.

[InstallScope]$Scope

)

begin {

trap {$PSCmdlet.ThrowTerminatingError($PSItem)}

# Setup the Destination repository

$defaultRepoPath = $(Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'powershell/Modules')

# Get the current PSModulePath

$PSModulePaths = $env:PSModulePath.Split([Path]::PathSeparator, [StringSplitOptions]::RemoveEmptyEntries)

#Clear the ModuleFastCache if -Update is specified to ensure fresh lookups of remote module availability

if ($Update) {

Clear-ModuleFastCache

}

if ($Scope -eq [InstallScope]::CurrentUser) {

$Destination = 'CurrentUser'

}

if (-not $Destination) {

$Destination = $defaultRepoPath

} elseif ($IsWindows -and $Destination -eq 'CurrentUser') {

$windowsDefaultDocumentsPath = Join-Path ([Environment]::GetFolderPath('MyDocuments')) 'PowerShell/Modules'

$Destination = $windowsDefaultDocumentsPath

# if CurrentUser and is on Windows, we do not need to update the PSModulePath or the user profile.

# this allows for a similar experience to Install-Module and Install-PSResource

$NoPSModulePathUpdate = $true

$NoProfileUpdate = $true

}

# Autocreate the default as a convenience, otherwise require the path to be present to avoid mistakes

if ($Destination -eq $defaultRepoPath -and -not (Test-Path $Destination)) {

if (Approve-Action 'Create Destination Folder' $Destination) {

New-Item -ItemType Directory -Path $Destination -Force | Out-Null

}

}

$Destination = Resolve-Path $Destination

if (-not $NoPSModulePathUpdate) {

if ($defaultRepoPath -ne $Destination -and $Destination -notin $PSModulePaths) {

Write-Warning 'Parameter -Destination is set to a custom path not in your current PSModulePath. We will add it to your PSModulePath for this session. You can suppress this behavior with the -NoPSModulePathUpdate switch.'

$NoProfileUpdate = $true

}

$addToPathParams = @{

Destination = $Destination

NoProfileUpdate = $NoProfileUpdate

}

if ($PSBoundParameters.ContainsKey('Confirm')) {

$addToPathParams.Confirm = $PSBoundParameters.Confirm

}

Add-DestinationToPSModulePath @addtoPathParams

}

#We want to maintain a single HttpClient for the life of the module. This isn't as big of a deal as it used to be but

#it is still a best practice.

if (-not $SCRIPT:__ModuleFastHttpClient -or $Source -ne $SCRIPT:__ModuleFastHttpClient.BaseAddress) {

$SCRIPT:__ModuleFastHttpClient = New-ModuleFastClient -Credential $Credential

if (-not $SCRIPT:__ModuleFastHttpClient) {

throw 'Failed to create ModuleFast HTTPClient. This is a bug'

}

}

$httpClient = $SCRIPT:__ModuleFastHttpClient

$cancelSource = [CancellationTokenSource]::new()

[HashSet[ModuleFastSpec]]$ModulesToInstall = @()

[List[ModuleFastInfo]]$installPlan = @()

}

process {

#We initialize and type the container list here because there is a bug where the ParameterSet is not correct in the begin block if the pipeline is used. Null conditional keeps it from being reinitialized

switch ($PSCmdlet.ParameterSetName) {

'Specification' {

foreach ($ModuleToInstall in $Specification) {

$isDuplicate = -not $ModulesToInstall.Add($ModuleToInstall)

if ($isDuplicate) {

Write-Warning "$ModuleToInstall was specified twice, skipping duplicate"

}

}

break

}

'ModuleFastInfo' {

foreach ($info in $ModuleFastInfo) {

$installPlan.Add($info)

}

break

}

'Path' {

$ModulesToInstall = ConvertFrom-RequiredSpec -RequiredSpecPath $Path -SpecFileType $SpecFileType

}

}

}

end {

trap {$PSCmdlet.ThrowTerminatingError($PSItem)}

try {

if (-not $installPlan) {

if ($ModulesToInstall.Count -eq 0 -and $PSCmdlet.ParameterSetName -eq 'Specification') {

Write-Verbose '🔎 No modules specified to install. Beginning SpecFile detection...'

$modulesToInstall = if ($CI -and (Test-Path $CILockFilePath)) {

Write-Debug "Found lockfile at $CILockFilePath. Using for specification evaluation and ignoring all others."

ConvertFrom-RequiredSpec -RequiredSpecPath $CILockFilePath -SpecFileType $SpecFileType

if ($Update) {

Write-Verbose "-Update was specified but a lockfile was found. Ignoring -Update and using lockfile specification."

$Update = $false

}

} else {

$specFiles = Find-RequiredSpecFile $PWD -CILockFileHint $CILockFilePath

if (-not $specFiles) {

Write-Warning "No specfiles found in $PWD. Please ensure you have a .requires.json or .requires.psd1 file in the current directory, specify a path with -Path, or define specifications with the -Specification parameter to skip this search."

}

foreach ($specfile in $specFiles) {

Write-Verbose "Found Specfile $specFile. Evaluating..."

ConvertFrom-RequiredSpec -RequiredSpecPath $specFile -SpecFileType $SpecFileType

}

}

}

if (-not $ModulesToInstall) {

throw [InvalidDataException]'No modules specifications found to evaluate.'

}

#If we do not have an explicit implementation plan, fetch it

#This is done so that Get-ModuleFastPlan | Install-ModuleFastPlan and Install-ModuleFastPlan have the same flow.

[ModuleFastInfo[]]$installPlan = if ($PSCmdlet.ParameterSetName -eq 'ModuleFastInfo') {

$ModulesToInstall.ToArray()

} else {

Write-Progress -Id 1 -Activity 'Install-ModuleFast' -Status 'Plan' -PercentComplete 1

Get-ModuleFastPlan -Specification $ModulesToInstall -HttpClient $httpClient -Source $Source -Update:$Update -PreRelease:$Prerelease.IsPresent -DestinationOnly:$DestinationOnly -Destination $Destination

}

}

if ($installPlan.Count -eq 0) {

$planAlreadySatisfiedMessage = "`u{2705} $($ModulesToInstall.count) Module Specifications have all been satisfied by installed modules. If you would like to check for newer versions remotely, specify -Update"

if ($WhatIfPreference) {

Write-Host -fore DarkGreen $planAlreadySatisfiedMessage

} else {

Write-Verbose $planAlreadySatisfiedMessage

}

return

}

#Unless Plan was specified, run the process (WhatIf will also short circuit).

#Plan is specified first so that WhatIf message will only show if Plan is not specified due to -or short circuit logic.

if ($Plan -or -not (Approve-Action $Destination "Install $($installPlan.Count) Modules")) {

if ($Plan) {

Write-Verbose "📑 -Plan was specified. Returning a plan including $($installPlan.Count) Module Specifications"

}

#TODO: Separate planned installs and dependencies. Can probably do this with a dependency flag on the ModuleInfo item and some custom formatting.

Write-Output $installPlan

} else {

Write-Progress -Id 1 -Activity 'Install-ModuleFast' -Status "Installing: $($installPlan.count) Modules" -PercentComplete 50

$installHelperParams = @{

ModuleToInstall = $installPlan

Destination = $Destination

CancellationToken = $cancelSource.Token

HttpClient = $httpClient

Update = $Update -or $PSCmdlet.ParameterSetName -eq 'ModuleFastInfo'

ThrottleLimit = $ThrottleLimit

}

$installedModules = Install-ModuleFastHelper @installHelperParams

Write-Progress -Id 1 -Activity 'Install-ModuleFast' -Completed

Write-Verbose "`u{2705} All required modules installed! Exiting."

if ($PassThru) {

Write-Output $installedModules

}

}

if ($CI) {

#FIXME: If a package was already installed, it doesn't show up in this lockfile.

Write-Verbose "Writing lockfile to $CILockFilePath"

[Dictionary[string, string]]$lockFile = @{}

$installPlan

| ForEach-Object {

$lockFile.Add($PSItem.Name, $PSItem.ModuleVersion)

}

$lockFile

| ConvertTo-Json -Depth 2

| Out-File -FilePath $CILockFilePath -Encoding UTF8

}

} finally {

$cancelSource.Dispose()

}

}

}

function New-ModuleFastClient {

param(

[PSCredential]$Credential

)

Write-Debug 'Creating new ModuleFast HTTP Client. This should only happen once!'

$ErrorActionPreference = 'Stop'

#SocketsHttpHandler is the modern .NET 5+ default handler for HttpClient.

$httpHandler = [SocketsHttpHandler]@{

#The max connections are only in case we end up using HTTP/1.1 instead of HTTP/2 for whatever reason. HTTP/2 will only use one connection (but multiple streams) per the spec unless EnableMultipleHttp2Connections is specified

MaxConnectionsPerServer = 10

#Reduce the amount of round trip confirmations by setting window size to 64MB. ModuleFast should primarily be used on reliable fast connections. Dynamic scaling will reduce this if needed.

InitialHttp2StreamWindowSize = 16777216

AutomaticDecompression = 'All'

}

$httpClient = [HttpClient]::new($httpHandler)

$httpClient.BaseAddress = $Source

#When in parallel some operations may take a significant amount of time to return

$httpClient.Timeout = [TimeSpan]::FromSeconds(30)

#If a credential was provided, use it as a basic auth credential

if ($Credential) {

$httpClient.DefaultRequestHeaders.Authorization = ConvertTo-AuthenticationHeaderValue $Credential

}

#This user agent is important, it indicates to pwsh.gallery that we want dependency-only metadata

#TODO: Do this with a custom header instead

$userHeaderAdded = $httpClient.DefaultRequestHeaders.UserAgent.TryParseAdd('ModuleFast (github.com/JustinGrote/ModuleFast)')

if (-not $userHeaderAdded) {

throw 'Failed to add User-Agent header to HttpClient. This is a bug'

}

#This will multiplex all queries over a single connection, minimizing TLS setup overhead

#Should also support HTTP/3 on newest PS versions

$httpClient.DefaultVersionPolicy = [HttpVersionPolicy]::RequestVersionOrHigher

#This should enable HTTP/3 on Win11 22H2+ (or linux with http3 library) and PS 7.2+

[void][AppContext]::SetSwitch('System.Net.SocketsHttpHandler.Http3Support', $true)

return $httpClient

}

function Get-ModuleFastPlan {

<#

.NOTES

THIS COMMAND IS DEPRECATED AND WILL NOT RECEIVE PARAMETER UPDATES. Please use Install-ModuleFast -Plan instead.

#>

[CmdletBinding()]

[OutputType([ModuleFastInfo[]])]

param(

#The module(s) to install. This can be a string, a ModuleSpecification, a hashtable with nuget version style (e.g. @{Name='test';Version='1.0'}), a hashtable with ModuleSpecification style (e.g. @{Name='test';RequiredVersion='1.0'}),

[Alias('Name')]

[Parameter(Position = 0, Mandatory, ValueFromPipeline)][ModuleFastSpec[]]$Specification,

#The repository to scan for modules. TODO: Multi-repo support

[string]$Source = 'https://pwsh.gallery/index.json',

#Whether to include prerelease modules in the request

[Switch]$Prerelease,

#By default we use in-place modules if they satisfy the version requirements. This switch will force a search for all latest modules

[Switch]$Update,

[PSCredential]$Credential,

[HttpClient]$HttpClient = $(New-ModuleFastClient -Credential $Credential),

[int]$ParentProgress,

[string]$Destination,

[switch]$DestinationOnly,

[CancellationToken]$CancellationToken

)

BEGIN {

trap {$PSCmdlet.ThrowTerminatingError($PSItem)}

$ErrorActionPreference = 'Stop'

[HashSet[ModuleFastSpec]]$modulesToResolve = @()

#We use this token to cancel the HTTP requests if the user hits ctrl-C without having to dispose of the HttpClient.

#We get a child so that a cancellation here does not affect any upstream commands

$cancelTokenSource = $CancellationToken ? [CancellationTokenSource]::CreateLinkedTokenSource($CancellationToken) : [CancellationTokenSource]::new()

$CancellationToken = $cancelTokenSource.Token

#We pass this splat to all our HTTP requests to cut down on boilerplate

$httpContext = @{

HttpClient = $httpClient

CancellationToken = $CancellationToken

}

}

PROCESS {

trap {$PSCmdlet.ThrowTerminatingError($PSItem)}

foreach ($spec in $Specification) {

if (-not $ModulesToResolve.Add($spec)) {

Write-Warning "$spec was specified twice, skipping duplicate"

}

}

}

END {

trap {$PSCmdlet.ThrowTerminatingError($PSItem)}

try {

# A deduplicated list of modules to install

[HashSet[ModuleFastInfo]]$modulesToInstall = @{}

# We use this as a fast lookup table for the context of the request

[Dictionary[Task, ModuleFastSpec]]$taskSpecMap = @{}

#We use this to track the tasks that are currently running

#We dont need this to be ConcurrentList because we only manipulate it in the "main" runspace.

[List[Task]]$currentTasks = @()

#This is used to track the highest candidate if -Update was specified to force a remote lookup. If the candidate is still the most valid after remote lookup we can skip it without hitting disk to read the manifest again.

[Dictionary[ModuleFastSpec, ModuleFastInfo]]$bestLocalCandidate = @{}

foreach ($moduleSpec in $ModulesToResolve) {

Write-Verbose "${moduleSpec}: Evaluating Module Specification"

$findLocalParams = @{

Update = $Update

BestCandidate = ([ref]$bestLocalCandidate)

}

if ($DestinationOnly) { $findLocalParams.ModulePaths = $Destination }

[ModuleFastInfo]$localMatch = Find-LocalModule @findLocalParams $moduleSpec

if ($localMatch) {

Write-Debug "${localMatch}: 🎯 FOUND satisfying version $($localMatch.ModuleVersion) at $($localMatch.Location). Skipping remote search."

#TODO: Capture this somewhere that we can use it to report in the deploy plan

continue

}

#If we get this far, we didn't find a manifest in this module path

Write-Debug "${moduleSpec}: 🔍 No installed versions matched the spec. Will check remotely."

$task = Get-ModuleInfoAsync @httpContext -Endpoint $Source -Name $moduleSpec.Name

$taskSpecMap[$task] = $moduleSpec

$currentTasks.Add($task)

}

[int]$tasksCompleteCount = 1

[int]$resolveTaskCount = $currentTasks.Count -as [Int]

do {

#The timeout here allow ctrl-C to continue working in PowerShell

#-1 is returned by WaitAny if we hit the timeout before any tasks completed

$noTasksYetCompleted = -1

[int]$thisTaskIndex = [Task]::WaitAny($currentTasks, 500)

if ($thisTaskIndex -eq $noTasksYetCompleted) { continue }

#The Plan whitespace is intentional so that it lines up with install progress using the compact format

Write-Progress -Id 1 -Activity 'Install-ModuleFast' -Status "Plan: Resolving $tasksCompleteCount/$resolveTaskCount Module Dependencies" -PercentComplete ((($tasksCompleteCount / $resolveTaskCount) * 50) + 1)

#TODO: This only indicates headers were received, content may still be downloading and we dont want to block on that.

#For now the content is small but this could be faster if we have another inner loop that WaitAny's on content

#TODO: Perform a HEAD query to see if something has changed

$completedTask = $currentTasks[$thisTaskIndex]

[ModuleFastSpec]$currentModuleSpec = $taskSpecMap[$completedTask]

if (-not $currentModuleSpec) {

throw 'Failed to find Module Specification for completed task. This is a bug.'

}

if ($currentModuleSpec.Guid -ne [Guid]::Empty) {

Write-Warning "${currentModuleSpec}: A GUID constraint was found in the module spec. ModuleSpec will currently only verify GUIDs after the module has been installed, so a plan may not be accurate. It is not recommended to match modules by GUID in ModuleFast, but instead verify package signatures for full package authenticity."

}

Write-Debug "${currentModuleSpec}: Processing Response"

# We use GetAwaiter so we get proper error messages back, as things such as network errors might occur here.

try {

$response = $completedTask.GetAwaiter().GetResult()

| ConvertFrom-Json

Write-Debug "${currentModuleSpec}: Received Response with $($response.Count) pages"

} catch {

$taskException = $PSItem.Exception.InnerException

#TODO: Rewrite this as a handle filter

if ($taskException -isnot [HttpRequestException]) { throw }

[HttpRequestException]$err = $taskException

if ($err.StatusCode -eq [HttpStatusCode]::NotFound) {

throw [InvalidOperationException]"${currentModuleSpec}: module was not found in the $Source repository. Check the spelling and try again."

}

#All other cases

$PSItem.ErrorDetails = "${currentModuleSpec}: Failed to fetch module $currentModuleSpec from $Source. Error: $PSItem"

throw $PSItem

}

if (-not $response.count) {

throw [InvalidDataException]"${currentModuleSpec}: invalid result received from $Source. This is probably a bug. Content: $response"

}

#If what we are looking for exists in the response, we can stop looking

#TODO: Type the responses and check on the type, not the existence of a property.

#TODO: This needs to be moved to a function so it isn't duplicated down in the "else" section below

$pageLeaves = $response.items.items

$pageLeaves | ForEach-Object {

if ($PSItem.packageContent -and -not $PSItem.catalogEntry.packagecontent) {

$PSItem.catalogEntry

| Add-Member -NotePropertyName 'PackageContent' -NotePropertyValue $PSItem.packageContent

}

}

$entries = $pageLeaves.catalogEntry

#Get the highest version that satisfies the requirement in the inlined index, if possible

$selectedEntry = if ($entries) {

#Sanity Check for Modules

if ('ItemType:Script' -in $entries[0].tags) {

throw [NotImplementedException]"${currentModuleSpec}: Script installations are currently not supported."

}

[SortedSet[NuGetVersion]]$inlinedVersions = $entries.version

foreach ($candidate in $inlinedVersions.Reverse()) {

#Skip Prereleases unless explicitly requested

if (($candidate.IsPrerelease -or $candidate.HasMetadata) -and -not ($currentModuleSpec.PreRelease -or $Prerelease)) {

Write-Debug "${moduleSpec}: skipping candidate $candidate because it is a prerelease and prerelease was not specified either with the -Prerelease parameter, by specifying a prerelease version in the spec, or adding a ! on the module name spec to indicate prerelease is acceptable."

continue

}

if ($currentModuleSpec.SatisfiedBy($candidate)) {

Write-Debug "${ModuleSpec}: Found satisfying version $candidate in the inlined index."

$matchingEntry = $entries | Where-Object version -EQ $candidate

if ($matchingEntry.count -gt 1) { throw 'Multiple matching Entries found for a specific version. This is a bug and should not happen' }

$matchingEntry

break

}

}

}

if ($selectedEntry.count -gt 1) { throw 'Multiple Entries Selected. This is a bug.' }

#Search additional pages if we didn't find it in the inlined ones

$selectedEntry ??= $(

Write-Debug "${currentModuleSpec}: not found in inlined index. Determining appropriate page(s) to query"

#If not inlined, we need to find what page(s) might have the candidate info we are looking for, starting with the highest numbered page first

$pages = $response.items

| Where-Object { -not $PSItem.items } #Get non-inlined pages

| Where-Object {

[VersionRange]$pageRange = [VersionRange]::new($PSItem.Lower, $true, $PSItem.Upper, $true, $null, $null)

return $currentModuleSpec.Overlap($pageRange)

}

| Sort-Object -Descending { [NuGetVersion]$PSItem.Upper }

if (-not $pages) {

throw [InvalidOperationException]"${currentModuleSpec}: a matching module was not found in the $Source repository that satisfies the requested version constraints. You may need to specify -PreRelease or adjust your version constraints."

}

Write-Debug "${currentModuleSpec}: Found $(@($pages).Count) additional pages that might match the query: $($pages.'@id' -join ',')"

#TODO: This is relatively slow and blocking, but we would need complicated logic to process it in the main task handler loop.

#I really should make a pipeline that breaks off tasks based on the type of the response.

#This should be a relatively rare query that only happens when the latest package isn't being resolved.

#Start with the highest potentially matching page and work our way down until we find a match.

foreach ($page in $pages) {

$response = (Get-ModuleInfoAsync @httpContext -Uri $page.'@id').GetAwaiter().GetResult() | ConvertFrom-Json

$pageLeaves = $response.items | ForEach-Object {

if ($PSItem.packageContent -and -not $PSItem.catalogEntry.packagecontent) {

$PSItem.catalogEntry

| Add-Member -NotePropertyName 'PackageContent' -NotePropertyValue $PSItem.packageContent

}

$PSItem

}

$entries = $pageLeaves.catalogEntry

#TODO: Dedupe as a function with above

if ($entries) {

[SortedSet[NuGetVersion]]$pageVersions = $entries.version

foreach ($candidate in $pageVersions.Reverse()) {

#Skip Prereleases unless explicitly requested

if (($candidate.IsPrerelease -or $candidate.HasMetadata) -and -not ($currentModuleSpec.PreRelease -or $Prerelease)) {

Write-Debug "Skipping candidate $candidate because it is a prerelease and prerelease was not specified either with the -Prerelease parameter or with a ! on the module name."

continue

}

if ($currentModuleSpec.SatisfiedBy($candidate)) {

Write-Debug "${currentModuleSpec}: Found satisfying version $candidate in the additional pages."

$matchingEntry = $entries | Where-Object version -EQ $candidate

if (-not $matchingEntry) { throw 'Multiple matching Entries found for a specific version. This is a bug and should not happen' }

$matchingEntry

break

}

}

}

#Candidate found, no need to process additional pages

if ($matchingEntry) { break }

}

)

if (-not $selectedEntry) {

throw [InvalidOperationException]"${currentModuleSpec}: a matching module was not found in the $Source repository that satisfies the version constraints. You may need to specify -PreRelease or adjust your version constraints."

}

if (-not $selectedEntry.PackageContent) { throw "No package location found for $($selectedEntry.PackageContent). This should never happen and is a bug" }

[ModuleFastInfo]$selectedModule = [ModuleFastInfo]::new(

$selectedEntry.id,

$selectedEntry.version,

$selectedEntry.PackageContent

)

if ($moduleSpec.Guid -and $moduleSpec.Guid -ne [Guid]::Empty) {

$selectedModule.Guid = $moduleSpec.Guid

}

#If -Update was specified, we need to re-check that none of the selected modules are already installed.

#TODO: Persist state of the local modules found to this point so we don't have to recheck.

if ($Update -and $bestLocalCandidate[$currentModuleSpec].ModuleVersion -eq $selectedModule.ModuleVersion) {

Write-Debug "${selectedModule}: ✅ -Update was specified and the best remote candidate matches what is locally installed, so we can skip this module."

#TODO: Fix the flow so this isn't stated twice

[void]$taskSpecMap.Remove($completedTask)

[void]$currentTasks.Remove($completedTask)

$tasksCompleteCount++

continue

}

#Check if we have already processed this item and move on if we have

if (-not $modulesToInstall.Add($selectedModule)) {

Write-Debug "$selectedModule already exists in the install plan. Skipping..."

#TODO: Fix the flow so this isn't stated twice

[void]$taskSpecMap.Remove($completedTask)

[void]$currentTasks.Remove($completedTask)

$tasksCompleteCount++

continue

}

Write-Verbose "${selectedModule}: Added to install plan"

# HACK: Pwsh doesn't care about target framework as of today so we can skip that evaluation

# TODO: Should it? Should we check for the target framework and only install if it matches?

$dependencyInfo = $selectedEntry.dependencyGroups.dependencies

#Determine dependencies and add them to the pending tasks

if ($dependencyInfo) {

# HACK: I should be using the Id provided by the server, for now I'm just guessing because

# I need to add it to the ComparableModuleSpec class

[List[ModuleFastSpec]]$dependencies = $dependencyInfo | ForEach-Object {

# Handle rare cases where range is not specified in the dependency

[VersionRange]$range = [string]::IsNullOrWhiteSpace($PSItem.range) ?

[VersionRange]::new() :

[VersionRange]::Parse($PSItem.range)

[ModuleFastSpec]::new($PSItem.id, $range)

}

Write-Debug "${currentModuleSpec}: has $($dependencies.count) additional dependencies: $($dependencies -join ', ')"

# TODO: Where loop filter maybe

[ModuleFastSpec[]]$dependenciesToResolve = $dependencies | Where-Object {

$dependency = $PSItem

# TODO: This dependency resolution logic should be a separate function

# Maybe ModulesToInstall should be nested/grouped by Module Name then version to speed this up, as it currently

# enumerates every time which shouldn't be a big deal for small dependency trees but might be a

# meaninful performance difference on a whole-system upgrade.

[HashSet[string]]$moduleNames = $modulesToInstall.Name

if ($dependency.Name -notin $ModuleNames) {

Write-Debug "$($dependency.Name): No modules with this name currently exist in the install plan. Resolving dependency..."

return $true

}

$modulesToInstall

| Where-Object Name -EQ $dependency.Name

| Sort-Object ModuleVersion -Descending

| ForEach-Object {

if ($dependency.SatisfiedBy($PSItem.ModuleVersion)) {

Write-Debug "Dependency $dependency satisfied by existing planned install item $PSItem"

return $false

}

}

Write-Debug "Dependency $($dependency.Name) is not satisfied by any existing planned install items. Resolving dependency..."

return $true

}

if (-not $dependenciesToResolve) {

Write-Debug "$moduleSpec has no remaining dependencies that need resolving"

continue

}

Write-Debug "Fetching info on remaining $($dependenciesToResolve.count) dependencies"

# We do this here rather than populate modulesToResolve because the tasks wont start until all the existing tasks complete

# TODO: Figure out a way to dedupe this logic maybe recursively but I guess a function would be fine too

foreach ($dependencySpec in $dependenciesToResolve) {

$findLocalParams = @{

Update = $Update

BestCandidate = ([ref]$bestLocalCandidate)

}

if ($DestinationOnly) { $findLocalParams.ModulePaths = $Destination }

[ModuleFastInfo]$localMatch = Find-LocalModule @findLocalParams $dependencySpec

if ($localMatch) {

Write-Debug "FOUND local module $($localMatch.Name) $($localMatch.ModuleVersion) at $($localMatch.Location.AbsolutePath) that satisfies $moduleSpec. Skipping..."

#TODO: Capture this somewhere that we can use it to report in the deploy plan

continue

} else {

Write-Debug "No local modules that satisfies dependency $dependencySpec. Checking Remote..."

}

Write-Debug "${currentModuleSpec}: Fetching dependency $dependencySpec"

#TODO: Do a direct version lookup if the dependency is a required version

$task = Get-ModuleInfoAsync @httpContext -Endpoint $Source -Name $dependencySpec.Name

$taskSpecMap[$task] = $dependencySpec

#Used to track progress as tasks can get removed

$resolveTaskCount++

$currentTasks.Add($task)

}

}

#Putting .NET methods in a try/catch makes errors in them terminating

try {

[void]$taskSpecMap.Remove($completedTask)

[void]$currentTasks.Remove($completedTask)

$tasksCompleteCount++

} catch {

throw

}

} while ($currentTasks.count -gt 0)

if ($modulesToInstall) { return $modulesToInstall }

} finally {

#Cancel any outstanding tasks if unexpected error occurs

$cancelTokenSource.Dispose()

}

}

}

function Clear-ModuleFastCache {

<#

.SYNOPSIS

Clears the ModuleFast HTTP Cache. This is useful if you are expecting a newer version of a module to be available.

#>

Write-Debug "Flushing ModuleFast Request Cache"

$SCRIPT:RequestCache.Dispose()

$SCRIPT:RequestCache = [MemoryCache]::new('PowerShell-ModuleFast-RequestCache')

}

#endregion Public

#region Private

function Install-ModuleFastHelper {

[CmdletBinding()]

param(

[Parameter(Mandatory)][ModuleFastInfo[]]$ModuleToInstall,

[string]$Destination,

[Parameter(Mandatory)][CancellationToken]$CancellationToken,

[HttpClient]$HttpClient,

[switch]$Update,

[int]$ThrottleLimit

)

BEGIN {

#We use this token to cancel the HTTP requests if the user hits ctrl-C without having to dispose of the HttpClient.

#We get a child so that a cancellation here does not affect any upstream commands

$cancelTokenSource = $CancellationToken ? [CancellationTokenSource]::CreateLinkedTokenSource($CancellationToken) : [CancellationTokenSource]::new()

$CancellationToken = $cancelTokenSource.Token

}

END {

$ErrorActionPreference = 'Stop'

try {

#Used to keep track of context with Tasks, because we dont have "await" style syntax like C#

[Dictionary[Task, hashtable]]$taskMap = @{}

[List[Task[Stream]]]$streamTasks = foreach ($module in $ModuleToInstall) {

$installPath = Join-Path $Destination $module.Name (Resolve-FolderVersion $module.ModuleVersion)

#TODO: Do a get-localmodule check here

$installIndicatorPath = Join-Path $installPath '.incomplete'

if (Test-Path $installIndicatorPath) {

Write-Warning "${module}: Incomplete installation found at $installPath. Will delete and retry."

Remove-Item $installPath -Recurse -Force

}

if (Test-Path $installPath) {

$existingManifestPath = try {

Resolve-Path (Join-Path $installPath "$($module.Name).psd1") -ErrorAction Stop

} catch [ActionPreferenceStopException] {

throw "${module}: Existing module folder found at $installPath but the manifest could not be found. This is likely a corrupted or missing module and should be fixed manually."

}

#TODO: Dedupe all import-powershelldatafile operations to a function ideally

$existingModuleMetadata = Import-ModuleManifest $existingManifestPath

$existingVersion = [NugetVersion]::new(

$existingModuleMetadata.ModuleVersion,

$existingModuleMetadata.privatedata.psdata.prerelease

)

#Do a prerelease evaluation

if ($module.ModuleVersion -eq $existingVersion) {

if ($Update) {

Write-Debug "${module}: Existing module found at $installPath and its version $existingVersion is the same as the requested version. -Update was specified so we are assuming that the discovered online version is the same as the local version and skipping this module installation."

continue

} else {

throw [NotImplementedException]"${module}: Existing module found at $installPath and its version $existingVersion is the same as the requested version. This is probably a bug because it should have been detected by localmodule detection. Use -Update to override..."

}

}

if ($module.ModuleVersion -lt $existingVersion) {

#TODO: Add force to override

throw [NotSupportedException]"${module}: Existing module found at $installPath and its version $existingVersion is newer than the requested prerelease version $($module.ModuleVersion). If you wish to continue, please remove the existing module folder or modify your specification and try again."

} else {

Write-Warning "${module}: Planned version $($module.ModuleVersion) is newer than existing prerelease version $existingVersion so we will overwrite."

Remove-Item $installPath -Force -Recurse

}

}

Write-Verbose "${module}: Downloading from $($module.Location)"

if (-not $module.Location) {

throw "${module}: No Download Link found. This is a bug"

}

$streamTask = $httpClient.GetStreamAsync($module.Location, $CancellationToken)

$context = @{

Module = $module

InstallPath = $installPath

}

$taskMap.Add($streamTask, $context)

$streamTask

}

[List[Job2]]$installJobs = while ($streamTasks.count -gt 0) {

$noTasksYetCompleted = -1

[int]$thisTaskIndex = [Task]::WaitAny($streamTasks, 500)

if ($thisTaskIndex -eq $noTasksYetCompleted) { continue }

$thisTask = $streamTasks[$thisTaskIndex]

$stream = $thisTask.GetAwaiter().GetResult()

$context = $taskMap[$thisTask]

$context.fetchStream = $stream

$streamTasks.RemoveAt($thisTaskIndex)

# This is a sync process and we want to do it in parallel, hence the threadjob

Write-Verbose "$($context.Module): Extracting to $($context.installPath)"

$installJob = Start-ThreadJob -ThrottleLimit $ThrottleLimit {

param(

[ValidateNotNullOrEmpty()]$stream = $USING:stream,

[ValidateNotNullOrEmpty()]$context = $USING:context

)

process {

try {

$installPath = $context.InstallPath

$installIndicatorPath = Join-Path $installPath '.incomplete'

if (Test-Path $installIndicatorPath) {

#FIXME: Output inside a threadjob is not surfaced to the user.

Write-Warning "$($context.Module): Incomplete installation found at $installPath. Will delete and retry."

Remove-Item $installPath -Recurse -Force

}

if (-not (Test-Path $context.InstallPath)) {

New-Item -Path $context.InstallPath -ItemType Directory -Force | Out-Null

}

New-Item -ItemType File -Path $installIndicatorPath -Force | Out-Null

#We are going to extract these straight out of memory, so we don't need to write the nupkg to disk

$zip = [IO.Compression.ZipArchive]::new($stream, 'Read')

[IO.Compression.ZipFileExtensions]::ExtractToDirectory($zip, $installPath)

if ($context.Module.Guid -and $context.Module.Guid -ne [Guid]::Empty) {

Write-Debug "$($context.Module): GUID was specified in Module. Verifying manifest"

$manifestPath = Join-Path $installPath "$($context.Module.Name).psd1"

#FIXME: This should be using Import-ModuleManifest but it needs to be brought in via the ThreadJob context. This will fail if the module has a dynamic manifest.

$manifest = Import-PowerShellDataFile $manifestPath

if ($manifest.Guid -ne $context.Module.Guid) {