-
Notifications
You must be signed in to change notification settings - Fork 182
Download installApps and installTestApps as part of "Download Project Dependencies" #2101
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
aholstrup1
wants to merge
30
commits into
microsoft:main
Choose a base branch
from
aholstrup1:aholstrup/installApps_download
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+751
−28
Open
Changes from all commits
Commits
Show all changes
30 commits
Select commit
Hold shift + click to select a range
23ff383
Initial implementation
aholstrup1 abd86aa
Add PS5 compatibility
aholstrup1 a0b3ecd
Remove duplicate check
aholstrup1 2a2e1cf
Remove logic from Runpipeline
aholstrup1 323d90a
releasenotes
aholstrup1 e992e7f
Check secret exists
aholstrup1 37a1b20
handle empty sanitizedFileName
aholstrup1 840a612
Handle duplicate files
aholstrup1 3bbd0d1
Error message
aholstrup1 b0d5fa4
releasenotes
aholstrup1 15554dc
Use Invoke-CommandWithRetry
aholstrup1 c74fa44
Fix for how install apps is set
aholstrup1 1c2a9d3
Handling for .zip files
aholstrup1 7c2cc34
Add tests
aholstrup1 afab810
Merge branch 'main' of https://github.com/microsoft/al-go into aholst…
aholstrup1 0b446c0
Cleanup
aholstrup1 50b19d5
Import Github-Helper.psm1
aholstrup1 cc992ad
Add handling of local paths
aholstrup1 cc03d43
Make sure all app files are always copied to dependencies folder
aholstrup1 0ba2ae6
Update test
aholstrup1 25d5c24
Fix compatibility issue
aholstrup1 8995cc8
Fix test
aholstrup1 488057f
debugging
aholstrup1 7d393ce
Check if folder exists
aholstrup1 8f660fe
Better logging
aholstrup1 8c6e34e
Fix trailing whitespace and missing blank line
aholstrup1 4b9cf82
remove out folder
aholstrup1 697edbb
Change directory
aholstrup1 ff159c2
Suggestions batch 1
aholstrup1 324518c
suggestions batch 2
aholstrup1 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
299 changes: 299 additions & 0 deletions
299
Actions/DownloadProjectDependencies/DownloadProjectDependencies.psm1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,299 @@ | ||
| Import-Module -Name (Join-Path $PSScriptRoot '../Github-Helper.psm1') | ||
| . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) | ||
|
|
||
| <# | ||
| .SYNOPSIS | ||
| Tests if a file is a ZIP archive by checking for the "PK" magic bytes. | ||
| .PARAMETER Path | ||
| The path to the file to test. | ||
| .OUTPUTS | ||
| $true if the file is a ZIP archive, $false otherwise. | ||
| #> | ||
| function Test-IsZipFile { | ||
| Param( | ||
| [string] $Path | ||
| ) | ||
| $extension = [System.IO.Path]::GetExtension($Path).ToLowerInvariant() | ||
| if ($extension -eq '.zip') { | ||
| return $true | ||
| } | ||
| # Check for ZIP magic bytes "PK" (0x50 0x4B) | ||
| # This handles the case where the file does not have a .zip extension but is still a ZIP archive (like .nupkg) | ||
| if ($PSVersionTable.PSVersion.Major -ge 6) { | ||
| $bytes = Get-Content -Path $Path -AsByteStream -TotalCount 2 -ErrorAction SilentlyContinue | ||
| } else { | ||
| $bytes = Get-Content -Path $Path -Encoding Byte -TotalCount 2 -ErrorAction SilentlyContinue | ||
| } | ||
| if ($bytes -and $bytes.Count -eq 2) { | ||
| return ([char]$bytes[0] -eq 'P') -and ([char]$bytes[1] -eq 'K') | ||
| } | ||
| return $false | ||
| } | ||
|
|
||
| <# | ||
| .SYNOPSIS | ||
| Extracts .app files from a ZIP archive to the destination path. | ||
| .PARAMETER ZipFile | ||
| The path to the ZIP file. | ||
| .PARAMETER DestinationPath | ||
| The path where .app files should be extracted to. | ||
| .OUTPUTS | ||
| An array of paths to the extracted .app files. | ||
| #> | ||
| function Expand-ZipFileToAppFiles { | ||
| Param( | ||
| [string] $ZipFile, | ||
| [string] $DestinationPath | ||
| ) | ||
| $fileName = [System.IO.Path]::GetFileName($ZipFile) | ||
| OutputDebug -message "Expanding zip file to extract .app files: $ZipFile" | ||
|
|
||
| # If file doesn't have .zip extension, copy to temp with .zip extension for Expand-Archive | ||
| $zipToExtract = $ZipFile | ||
| $tempZipCreated = $false | ||
| if ([System.IO.Path]::GetExtension($ZipFile).ToLowerInvariant() -ne '.zip') { | ||
| $zipToExtract = Join-Path (GetTemporaryPath) "$([System.IO.Path]::GetFileName($ZipFile)).zip" | ||
| Copy-Item -Path $ZipFile -Destination $zipToExtract | ||
| $tempZipCreated = $true | ||
| } | ||
|
|
||
| try { | ||
| # Extract to runner temp folder | ||
| $extractPath = Join-Path (GetTemporaryPath) ([System.IO.Path]::GetFileNameWithoutExtension($fileName)) | ||
| Expand-Archive -Path $zipToExtract -DestinationPath $extractPath -Force | ||
|
|
||
| # Find all files in the extracted folder and process them | ||
| $appFiles = @() | ||
| foreach ($file in (Get-ChildItem -Path $extractPath -Recurse -File)) { | ||
| $extension = [System.IO.Path]::GetExtension($file.FullName).ToLowerInvariant() | ||
|
|
||
| if ($extension -eq '.app') { | ||
| $destFile = Join-Path $DestinationPath $file.Name | ||
| Copy-Item -Path $file.FullName -Destination $destFile -Force | ||
| $appFiles += $destFile | ||
| } | ||
| elseif (Test-IsZipFile -Path $file.FullName) { | ||
| # Recursively extract nested ZIP files | ||
| $appFiles += Expand-ZipFileToAppFiles -ZipFile $file.FullName -DestinationPath $DestinationPath | ||
| } | ||
| } | ||
|
|
||
| # Clean up the extracted folder | ||
| Remove-Item -Path $extractPath -Recurse -Force | ||
|
|
||
| if ($appFiles.Count -eq 0) { | ||
| OutputWarning -message "No .app files found in zip archive: $fileName" | ||
| } else { | ||
| OutputDebug -message "Found $($appFiles.Count) .app file(s) in zip archive" | ||
| } | ||
| return $appFiles | ||
| } | ||
| finally { | ||
| if ($tempZipCreated) { | ||
| Remove-Item -Path $zipToExtract -Force -ErrorAction SilentlyContinue | ||
| } | ||
| } | ||
| } | ||
|
|
||
| <# | ||
| .SYNOPSIS | ||
| Resolves a local path to an array of .app file paths. | ||
| .DESCRIPTION | ||
| Handles local files and folders: | ||
| - If path is an .app file: returns it | ||
| - If path is a folder: recursively finds all .app files | ||
| - If path contains wildcards: resolves them to matching files | ||
| - If path is a ZIP file (by extension or magic bytes): extracts and returns .app files | ||
| .PARAMETER Path | ||
| The local file or folder path. | ||
| .PARAMETER DestinationPath | ||
| The path where extracted .app files should be placed (for ZIP files). | ||
| .OUTPUTS | ||
| An array of paths to .app files. | ||
| #> | ||
| function Get-AppFilesFromLocalPath { | ||
| Param( | ||
| [string] $Path, | ||
| [string] $DestinationPath | ||
| ) | ||
|
|
||
| # Ensure the destination directory exists | ||
| if (-not (Test-Path -Path $DestinationPath)) { | ||
| New-Item -ItemType Directory -Path $DestinationPath -Force | Out-Null | ||
| } | ||
|
|
||
| # Get all matching items (works for folders, wildcards, and single files) | ||
| $matchedItems = @(Get-ChildItem -Path $Path -Recurse -File -ErrorAction SilentlyContinue) | ||
|
|
||
| if ($matchedItems.Count -eq 0) { | ||
| OutputWarning -message "No files found at local path: $Path" | ||
| return @() | ||
| } | ||
|
|
||
| # Process each matched file | ||
| $appFiles = @() | ||
| foreach ($item in $matchedItems) { | ||
| $extension = [System.IO.Path]::GetExtension($item.FullName).ToLowerInvariant() | ||
|
|
||
| if ($extension -eq '.app') { | ||
| $destFile = Join-Path $DestinationPath $item.Name | ||
| Copy-Item -Path $item.FullName -Destination $destFile -Force | ||
| $appFiles += $destFile | ||
| } elseif (Test-IsZipFile -Path $item.FullName) { | ||
| $appFiles += Expand-ZipFileToAppFiles -ZipFile $item.FullName -DestinationPath $DestinationPath | ||
| } else { | ||
| OutputWarning -message "Unknown file type for local path: $($item.FullName). Skipping." | ||
| } | ||
| } | ||
| return $appFiles | ||
| } | ||
|
|
||
| <# | ||
| .SYNOPSIS | ||
| Downloads a file from a URL to a specified download path. | ||
| .DESCRIPTION | ||
| Downloads a file from a URL to a specified download path. | ||
| It handles URL decoding and sanitizes the file name. | ||
| If the downloaded file is a zip file, it extracts the .app files from it. | ||
| .PARAMETER Url | ||
| The URL of the file to download. | ||
| .PARAMETER CleanUrl | ||
| The original URL for error reporting. | ||
| .PARAMETER DownloadPath | ||
| The path where the file should be downloaded. | ||
| .OUTPUTS | ||
| An array of paths to the downloaded/extracted .app files. | ||
| #> | ||
| function Get-AppFilesFromUrl { | ||
| Param( | ||
| [string] $Url, | ||
| [string] $CleanUrl, | ||
| [string] $DownloadPath | ||
| ) | ||
|
|
||
| # Ensure the download directory exists | ||
| if (-not (Test-Path -Path $DownloadPath)) { | ||
| New-Item -ItemType Directory -Path $DownloadPath -Force | Out-Null | ||
| } | ||
|
|
||
| # Get the file name from the URL | ||
| $urlWithoutQuery = $Url.Split('?')[0].TrimEnd('/') | ||
| $rawFileName = [System.IO.Path]::GetFileName($urlWithoutQuery) | ||
| $decodedFileName = [Uri]::UnescapeDataString($rawFileName) | ||
| $decodedFileName = [System.IO.Path]::GetFileName($decodedFileName) | ||
|
|
||
| # Sanitize file name by removing invalid characters | ||
| $sanitizedFileName = $decodedFileName.Split([System.IO.Path]::getInvalidFileNameChars()) -join "" | ||
| $sanitizedFileName = $sanitizedFileName.Trim() | ||
|
|
||
| if ([string]::IsNullOrWhiteSpace($sanitizedFileName)) { | ||
| # Assume the file is an .app file if no valid name could be determined | ||
| $sanitizedFileName = "$([Guid]::NewGuid().ToString()).app" | ||
| } | ||
|
|
||
| # Get the final file path | ||
| $downloadedFile = Join-Path $DownloadPath $sanitizedFileName | ||
| if (Test-Path -LiteralPath $downloadedFile) { | ||
| OutputWarning -message "Overwriting existing file '$sanitizedFileName'. Multiple dependencies may resolve to the same filename." | ||
| } | ||
|
|
||
| # Download with retry logic | ||
| try { | ||
| Invoke-CommandWithRetry -ScriptBlock { | ||
| Invoke-WebRequest -Method GET -UseBasicParsing -Uri $Url -OutFile $downloadedFile | Out-Null | ||
| } -RetryCount 3 -FirstDelay 5 -MaxWaitBetweenRetries 10 | ||
| OutputDebug -message "Downloaded file to path: $downloadedFile" | ||
| } catch { | ||
| throw "Failed to download file from inaccessible URL: $CleanUrl. Error was: $($_.Exception.Message)" | ||
| } | ||
|
|
||
| # Check if the downloaded file is a zip file (by extension or magic bytes) | ||
| if (Test-IsZipFile -Path $downloadedFile) { | ||
| $appFiles = Expand-ZipFileToAppFiles -ZipFile $downloadedFile -DestinationPath $DownloadPath | ||
| Remove-Item -Path $downloadedFile -Force | ||
| return $appFiles | ||
| } | ||
|
|
||
| return @($downloadedFile) | ||
| } | ||
|
|
||
| <# | ||
| .SYNOPSIS | ||
| Downloads dependencies from URLs specified in installApps and installTestApps settings. | ||
| .DESCRIPTION | ||
| Reads the installApps and installTestApps arrays from the repository settings. | ||
| For each entry that is a URL (starts with http:// or https://): | ||
| - Resolves any secret placeholders in the format ${{ secretName }} by looking up the secret value | ||
| - Downloads the app file to the specified destination path | ||
| For entries that are local paths: | ||
| - Resolves folders to their contained .app files | ||
| - Extracts .app files from ZIP archives | ||
| .PARAMETER DestinationPath | ||
| The path where the app files should be downloaded. | ||
| .OUTPUTS | ||
| A hashtable with Apps and TestApps arrays containing the resolved local file paths. | ||
| #> | ||
| function Get-DependenciesFromInstallApps { | ||
| Param( | ||
| [string] $DestinationPath | ||
| ) | ||
|
|
||
| $settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable | ||
|
|
||
| # ENV:Secrets is not set when running Pull_Request trigger | ||
| if ($env:Secrets) { | ||
| $secrets = $env:Secrets | ConvertFrom-Json | ConvertTo-HashTable | ||
| } | ||
| else { | ||
| $secrets = @{} | ||
| } | ||
|
|
||
| # Initialize the install hashtable | ||
| $install = @{ | ||
| "Apps" = @($settings.installApps) | ||
| "TestApps" = @($settings.installTestApps) | ||
| } | ||
|
|
||
| # Check if the installApps and installTestApps settings are empty | ||
| if (($settings.installApps.Count -eq 0) -and ($settings.installTestApps.Count -eq 0)) { | ||
| Write-Host "No installApps or installTestApps settings found." | ||
| return $install | ||
| } | ||
|
|
||
| # Replace secret names in install.apps and install.testApps and download files from URLs | ||
| foreach($list in @('Apps','TestApps')) { | ||
|
|
||
| $updatedListOfFiles = @() | ||
| foreach($appFile in $install."$list") { | ||
| Write-Host "Processing install$($list) entry: $appFile" | ||
|
|
||
| # If the app file is not a URL, resolve local path. | ||
| if ($appFile -notlike 'http*://*') { | ||
| $updatedListOfFiles += Get-AppFilesFromLocalPath -Path $appFile -DestinationPath $DestinationPath | ||
| } else { | ||
| # Else, check for secrets in the URL and replace them | ||
| $appFileUrl = $appFile | ||
| $pattern = '.*(\$\{\{\s*([^}]+?)\s*\}\}).*' | ||
| if ($appFile -match $pattern) { | ||
| $secretName = $matches[2] | ||
| if (-not $secrets.ContainsKey($secretName)) { | ||
| throw "Setting: install$($list) references unknown secret '$secretName' in URL: $appFile" | ||
| } | ||
| $appFileUrl = $appFileUrl.Replace($matches[1],[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$secretName"))) | ||
| } | ||
|
|
||
| # Download the file (may return multiple .app files if it's a zip) | ||
| $appFiles = Get-AppFilesFromUrl -Url $appFileUrl -CleanUrl $appFile -DownloadPath $DestinationPath | ||
|
|
||
| $updatedListOfFiles += $appFiles | ||
| } | ||
| } | ||
|
|
||
| # Update the install hashtable with the resolved file paths | ||
| $install."$list" = $updatedListOfFiles | ||
| } | ||
|
|
||
| return $install | ||
| } | ||
|
|
||
| Export-ModuleMember -Function Get-AppFilesFromUrl, Get-AppFilesFromLocalPath, Get-DependenciesFromInstallApps | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.