Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.github/workflows/*.lock.yml linguist-generated=true
9 changes: 9 additions & 0 deletions .github/aw/actions-lock.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"entries": {
"github/gh-aw-actions/setup@v1.0.1": {
"repo": "github/gh-aw-actions/setup",
"version": "v1.0.1",
"sha": "221ba4f4319de6b8e809d1a15528b3ae6a818aa5"
}
}
}
79 changes: 79 additions & 0 deletions .github/workflows/psgallery-health-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# PowerShell Gallery Health Check
#
# Cheap, deterministic probe of the PowerShell Gallery endpoints that PSResourceGet depends on.
# A probe fails only if it keeps returning 404 or 5xx across several retries. When a failure is
# confirmed, this workflow dispatches the 'psgallery-outage-reporter' agentic workflow, which
# opens (or updates) a service-status issue in PowerShell/PowerShellGallery so the status page
# at https://aka.ms/psgallery can be updated.
name: 'PowerShell Gallery Health Check'

on:
schedule:
- cron: '*/30 * * * *'
workflow_dispatch:

permissions:
contents: read

concurrency:
group: psgallery-health-check
cancel-in-progress: false

jobs:
probe:
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
actions: write
steps:
- name: 'Probe PowerShell Gallery endpoints'
id: probe
shell: pwsh
run: |
$probes = [ordered]@{
'V2 service root' = 'https://www.powershellgallery.com/api/v2/'
'Find by name' = "https://www.powershellgallery.com/api/v2/FindPackagesById()?id='PowerShellGet'"
'Find specific version'= "https://www.powershellgallery.com/api/v2/Packages(Id='PowerShellGet',Version='2.2.5')"
'Search' = "https://www.powershellgallery.com/api/v2/Search()?searchTerm='PSReadLine'&includePrerelease=false&`$top=1"
'Package download' = 'https://www.powershellgallery.com/api/v2/package/PowerShellGet/2.2.5'
}

$maxAttempts = 3
$failures = @()
foreach ($name in $probes.Keys) {
$uri = $probes[$name]
$status = 0
for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) {
$status = [int](curl --silent --location --output /dev/null --max-time 60 --write-out '%{http_code}' $uri)
if ($status -ne 404 -and $status -lt 500) { break }
if ($attempt -lt $maxAttempts) { Start-Sleep -Seconds 30 }
}

Write-Host "$name -> HTTP $status ($uri)"
if ($status -eq 404 -or $status -ge 500) {
$failures += [pscustomobject]@{
probe = $name
uri = $uri
statusCode = $status
attempts = $maxAttempts
checkedAt = (Get-Date).ToUniversalTime().ToString('o')
}
}
}

$json = if ($failures.Count) { ConvertTo-Json -InputObject $failures -Compress -Depth 3 } else { '[]' }
"failed=$([bool]$failures.Count)".ToLower() >> $env:GITHUB_OUTPUT
"results=$json" >> $env:GITHUB_OUTPUT

- name: 'Dispatch outage reporter'
if: steps.probe.outputs.failed == 'true'
env:
GH_TOKEN: ${{ github.token }}
PROBE_RESULTS: ${{ steps.probe.outputs.results }}
run: |
gh workflow run psgallery-outage-reporter.lock.yml \
--repo "${{ github.repository }}" \
--ref "${{ github.ref_name }}" \
-f probe_results="$PROBE_RESULTS" \
-f health_check_run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
2,143 changes: 2,143 additions & 0 deletions .github/workflows/psgallery-outage-reporter.lock.yml

Large diffs are not rendered by default.

102 changes: 102 additions & 0 deletions .github/workflows/psgallery-outage-reporter.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
---
description: |
Reports PowerShell Gallery outages. Triggered by the 'PowerShell Gallery Health Check'
workflow when gallery endpoints keep returning 404 or 5xx. Re-verifies the failure, then
opens (or updates) a service-status issue in PowerShell/PowerShellGallery so the status
page at https://aka.ms/psgallery can be updated.

on:
workflow_dispatch:
inputs:
probe_results:
description: 'JSON array of failing probes reported by the health check'
required: true
type: string
health_check_run_url:
description: 'URL of the health check run that detected the failure'
required: false
type: string

permissions:
contents: read
issues: read

engine: copilot

timeout-minutes: 15

network:
allowed:
- defaults
- powershell

tools:
github:
toolsets: [issues]
bash: ["curl:*", "jq:*", "sleep", "date"]

safe-outputs:
# PAT or GitHub App token with 'issues: write' on PowerShell/PowerShellGallery.
github-token: ${{ secrets.PSGALLERY_STATUS_ISSUE_TOKEN }}
create-issue:
target-repo: "PowerShell/PowerShellGallery"
title-prefix: "[Service Status] "
max: 1
add-comment:
target: "*"
target-repo: "PowerShell/PowerShellGallery"
max: 1
---

# PowerShell Gallery outage reporter

You are an on-call assistant for PSResourceGet. The scheduled health check detected that one or
more PowerShell Gallery endpoints repeatedly returned HTTP 404 or 5xx responses.

Failing probes reported by the health check (JSON):

```json
${{ github.event.inputs.probe_results }}
```

Health check run: ${{ github.event.inputs.health_check_run_url }}

## Steps

1. **Re-verify.** For every failing probe, request its `uri` again with
`curl --silent --location --output /dev/null --max-time 60 --write-out '%{http_code}' '<uri>'`,
Comment on lines +66 to +67
up to 3 times, running `sleep 20` between attempts. Treat a probe as still failing only if every
attempt returns 404 or a status code of 500 or higher. If no probe is still failing, the incident
has recovered: do **not** create an issue or comment; finish by reporting that no action was needed.

Shell commands are restricted to an allowlist: run each `curl`, `sleep`, and `date` invocation as
its own separate command. Do not use loops, pipes, `;`, `&&`, subshells, or variables.

Every probe URI targets a package or endpoint that is expected to exist, so a 404 is a failure.
Do not second-guess whether a 404 is "expected" based on the package name.

2. **Look for an existing report.** Search open issues in `PowerShell/PowerShellGallery` whose
title starts with `[Service Status]` and that were created in the last 24 hours. Also consider
any open issue that clearly describes the same outage (same endpoints / status codes).

3. **Report.**
- If a matching open issue exists, add **one** comment to it with the latest re-verification
results (timestamp in UTC, endpoint, status code). Keep it short.
- Otherwise create **one** issue in `PowerShell/PowerShellGallery`:
- Title: a short summary such as `PowerShell Gallery returning HTTP 503 for package downloads`
(the `[Service Status]` prefix is added automatically).
- Body must include:
- A one-paragraph summary of the impact on users (for example: `Find-PSResource` /
`Install-PSResource` / `Save-PSResource` against `PSGallery` fail).
- A Markdown table of still-failing probes: probe, URI, HTTP status, time checked (UTC).
- Any probes from the health check that recovered during re-verification.
- A link to the health check run (above).
- A request to the PowerShell Gallery team to update the service status at
[aka.ms/psgallery](https://aka.ms/psgallery) and to close this issue once resolved.
- A note that this issue was opened automatically by the PSResourceGet health monitor.

## Rules

- Never include secrets, tokens, or request headers in the issue or comment.
- Never create more than one issue or more than one comment per run.
- Do not speculate about root cause; report only what you observed.
5 changes: 5 additions & 0 deletions src/code/FindHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -924,7 +924,12 @@ private IEnumerable<PSResourceInfo> SearchByNames(ServerApiCall currentServer, R

responses = response.GetAwaiter().GetResult();

bool requestFailed = !errorMsgs.IsEmpty;
Utils.WriteOutConcurrentQueue(_cmdletPassedIn, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
if (requestFailed)
{
continue;
}
}
else
{
Expand Down
31 changes: 31 additions & 0 deletions src/code/Utils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,37 @@ public static string FormatRequestsExceptions(Exception exception, HttpRequestMe
return exMsg;
}

public const string PSGalleryStatusUri = "https://aka.ms/psgallery";

/// <summary>
/// Formats an HTTP request exception and, when the request targeted the PowerShell Gallery and failed
/// with a 404 or 5xx status code, appends a pointer to the PowerShell Gallery status page.
/// </summary>
public static string FormatRequestsExceptions(Exception exception, HttpRequestMessage request, HttpStatusCode statusCode)
{
string exMsg = FormatRequestsExceptions(exception, request);
if (IsPSGalleryServiceError(request?.RequestUri, statusCode))
{
exMsg += $" The PowerShell Gallery may be experiencing an issue. Check {PSGalleryStatusUri} for the current service status.";
}

return exMsg;
}

public static bool IsPSGalleryServiceError(Uri requestUri, HttpStatusCode statusCode)
{
if (requestUri == null || !requestUri.IsAbsoluteUri)
{
return false;
}

string host = requestUri.Host;
bool isPSGallery = host.Equals("powershellgallery.com", StringComparison.OrdinalIgnoreCase) ||
host.EndsWith(".powershellgallery.com", StringComparison.OrdinalIgnoreCase);

return isPSGallery && (statusCode == HttpStatusCode.NotFound || (int)statusCode >= 500);
}

public static string FormatCredentialRequestExceptions(Exception exception)
{
string exMsg = $"'{exception.Message}' Re-run the command with -Credential.";
Expand Down
32 changes: 23 additions & 9 deletions src/code/V2ServerAPICalls.cs
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,7 @@ public override async Task<FindResults> FindNameAsync(string packageName, bool i

var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}";
string response = string.Empty;
int errorCountBeforeRequest = errorMsgs.Count;
try
{
response = await HttpRequestCallAsync(requestUrlV2, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
Expand Down Expand Up @@ -476,6 +477,12 @@ public override async Task<FindResults> FindNameAsync(string packageName, bool i
}
}

// HttpRequestCallAsync reports failures through errorMsgs rather than throwing; don't also report a parse/not-found error.
if (string.IsNullOrEmpty(response) && errorMsgs.Count > errorCountBeforeRequest)
{
return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType);
}

int count = GetCountFromResponse(response, out string errStr);
if (errStr != null)
{
Expand Down Expand Up @@ -833,6 +840,7 @@ public override async Task<FindResults> FindVersionAsync(string packageName, str

var requestUrlV2 = $"{Repository.Uri}/FindPackagesById()?{queryBuilder.BuildQueryString()}";
string response = string.Empty;
int errorCountBeforeRequest = errorMsgs.Count;
try
{
response = await HttpRequestCallAsync(requestUrlV2, errorMsgs, warningMsgs, debugMsgs, verboseMsgs);
Expand Down Expand Up @@ -874,6 +882,12 @@ public override async Task<FindResults> FindVersionAsync(string packageName, str
}
}

// HttpRequestCallAsync reports failures through errorMsgs rather than throwing; don't also report a parse/not-found error.
if (string.IsNullOrEmpty(response) && errorMsgs.Count > errorCountBeforeRequest)
{
return new FindResults(stringResponse: Utils.EmptyStrArray, hashtableResponse: emptyHashResponses, responseType: v2FindResponseType);
}

int count = GetCountFromResponse(response, out string errStr);
if (errStr != null)
{
Expand Down Expand Up @@ -1087,23 +1101,23 @@ private async Task<string> HttpRequestCallAsync(string requestUrlV2, ConcurrentQ
catch (ResourceNotFoundException e)
{
errorMsgs.Enqueue(new ErrorRecord(
new ResourceNotFoundException($"Failure when attempting to make request for '{requestUrlV2}'", e),
new ResourceNotFoundException($"Failure when attempting to make request for '{requestUrlV2}'. {e.Message}", e),
"ResourceNotFound",
ErrorCategory.InvalidResult,
this));
}
catch (UnauthorizedException e)
{
errorMsgs.Enqueue(new ErrorRecord(
new UnauthorizedException($"Failure when attempting to make request for '{requestUrlV2}'", e),
new UnauthorizedException($"Failure when attempting to make request for '{requestUrlV2}'. {e.Message}", e),
"UnauthorizedRequest",
ErrorCategory.InvalidResult,
this));
}
catch (HttpRequestException e)
{
errorMsgs.Enqueue(new ErrorRecord(
new HttpRequestException($"Failure when attempting to make request for '{requestUrlV2}'", e),
new HttpRequestException($"Failure when attempting to make request for '{requestUrlV2}'. {e.Message}", e),
"HttpRequestCallFailure",
ErrorCategory.ConnectionError,
this));
Expand Down Expand Up @@ -1143,7 +1157,7 @@ private async Task<HttpContent> HttpRequestCallForContentAsync(string requestUrl
catch (HttpRequestException e)
{
errorMsgs.Enqueue(new ErrorRecord(
new HttpRequestException($"Failure when attempting to make request for content with '{requestUrlV2}'", e),
new HttpRequestException($"Failure when attempting to make request for content with '{requestUrlV2}'. {e.Message}", e),
"HttpRequestFailure",
ErrorCategory.ConnectionError,
this));
Expand Down Expand Up @@ -2027,7 +2041,7 @@ public static async Task<string> SendV2RequestAsync(HttpRequestMessage message,
{
if (responseStatusCode.Equals(HttpStatusCode.NotFound))
{
throw new ResourceNotFoundException(Utils.FormatRequestsExceptions(e, message));
throw new ResourceNotFoundException(Utils.FormatRequestsExceptions(e, message, responseStatusCode));
}
// ADO feed will return a 401 if a package does not exist on the feed, with the following message:
// 401 (Unauthorized - No local versions of package 'NonExistentModule'; please provide authentication to access
Expand All @@ -2036,13 +2050,13 @@ public static async Task<string> SendV2RequestAsync(HttpRequestMessage message,
{
if (e.Message.Contains("access versions from upstream that have not yet been saved to your feed"))
{
throw new ResourceNotFoundException(Utils.FormatRequestsExceptions(e, message));
throw new ResourceNotFoundException(Utils.FormatRequestsExceptions(e, message, responseStatusCode));
}

throw new UnauthorizedException(Utils.FormatCredentialRequestExceptions(e));
}

throw new HttpRequestException(Utils.FormatRequestsExceptions(e, message));
throw new HttpRequestException(Utils.FormatRequestsExceptions(e, message, responseStatusCode));
}
catch (ArgumentNullException e)
{
Expand Down Expand Up @@ -2072,14 +2086,14 @@ public static async Task<HttpContent> SendV2RequestForContentAsync(HttpRequestMe
{
if (responseStatusCode.Equals(HttpStatusCode.NotFound))
{
throw new ResourceNotFoundException(Utils.FormatRequestsExceptions(e, message));
throw new ResourceNotFoundException(Utils.FormatRequestsExceptions(e, message, responseStatusCode));
}
if (responseStatusCode.Equals(HttpStatusCode.Unauthorized))
{
throw new UnauthorizedException(Utils.FormatCredentialRequestExceptions(e));
}

throw new HttpRequestException(Utils.FormatRequestsExceptions(e, message));
throw new HttpRequestException(Utils.FormatRequestsExceptions(e, message, responseStatusCode));
}
catch (ArgumentNullException e)
{
Expand Down
Loading
Loading