Fetch organization packages with Workload Identity

CHANGELOG.md

@@ -8,7 +8,7 @@
 
 ### Enhancements
 
-* Support Workload Identity, sometimes also known as Trusted Publishing, in `mix hex.publish`. In a GitHub Actions job with the `id-token: write` permission and no `HEX_API_KEY` or other Hex credential configured, the job's OIDC token is exchanged for a short-lived token that can only publish the package, so CI can publish without storing an API key. A workload identity must be configured on hex.pm first, on the package for a public package and on the organization for a package in an organization's repository. An organization's workload identity with the `write` role can also create packages
+* Support Workload Identity, sometimes also known as Trusted Publishing, so a GitHub Actions job with the `id-token: write` permission can publish packages and fetch an organization's packages without storing a key. When no `HEX_API_KEY` or other Hex credential is configured, `mix hex.publish` exchanges the job's OIDC token for a short-lived token that can only publish the package. When no organization key or authenticated user is configured, fetching packages from an organization's repository exchanges it for a short-lived token that can fetch them. Configure a workload identity on hex.pm first, on the package for a public package and on the organization for a package in an organization's repository. An organization's workload identity with the `read` role can fetch its packages, and one with the `write` role can also publish and create packages
 * Print warnings to standard error instead of standard output, keeping stdout clean for machine-readable output such as `mix hex.outdated --json`. Warning-colored lines that are part of a command's regular output, such as retirement notices in `mix hex.info` and the `mix deps.get` dependency listing, remain on standard output
 * Add `--format sarif` and `--output PATH` options to `mix hex.audit` to render the audit result as a SARIF v2.1.0 document that can be uploaded to GitHub code scanning and other SARIF consumers. Findings are anchored to the dependency's `mix.lock` entry and ignored findings are included as suppressed results. Requires OTP 27 or later
 * Link to the hex.pm diffs page (`https://hex.pm/diffs`) in `mix hex.outdated` now that package diffs have moved from diff.hex.pm into hex.pm

lib/hex/auth.ex

@@ -66,6 +66,8 @@ defmodule Hex.Auth do
       persist_oauth_tokens: &persist_oauth_tokens/4,
       clear_oauth_tokens: &clear_oauth_tokens/0,
       organization_reauth: &organization_reauth/1,
+      get_workload_identity_token: &get_workload_identity_token/1,
+      persist_workload_identity_token: &persist_workload_identity_token/2,
       prompt_otp: &prompt_otp/1,
       get_client_id: &Hex.API.OAuth.client_id/0,
       should_authenticate: &should_authenticate/1
@@ -161,6 +163,18 @@ defmodule Hex.Auth do
   defp put_organization_reauth(token_data, organizations),
     do: Map.put(token_data, :organization_reauth_required, organizations)
 
+  # The outcome of a repository's Workload Identity exchange, a token or why
+  # there is none, is kept in memory only. Tokens expire in 15 minutes and come
+  # from the CI job rather than from anything the user configured, so they
+  # don't belong in the Hex config.
+  defp get_workload_identity_token(repo) do
+    Map.get(Hex.State.fetch!(:workload_identity_tokens), repo, :error)
+  end
+
+  defp persist_workload_identity_token(repo, result) do
+    Hex.State.update!(:workload_identity_tokens, &Map.put(&1, repo, result))
+  end
+
   # A prompt answers :eof when there is nothing on stdin to read, which is what
   # an OTP challenge in CI gets.
   defp prompt_otp(message) do

lib/hex/registry/server.ex

@@ -600,7 +600,13 @@ defmodule Hex.Registry.Server do
     end
 
     if not (missing? or unauthorized?) or Mix.debug?() do
-      Hex.Utils.print_error_result(result)
+      case result do
+        {:error, {:auth_error, {:workload_identity_failed, reason}}} ->
+          Hex.Shell.error(Hex.WorkloadIdentity.repository_error_message(repo, reason))
+
+        _other ->
+          Hex.Utils.print_error_result(result)
+      end
     end
   end
 
@@ -637,6 +643,9 @@ defmodule Hex.Registry.Server do
               "Set HEX_UNSAFE_REGISTRY=1 to disable this check and allow insecure package downloads."
           )
 
+        {:error, {:auth_error, {:workload_identity_failed, reason}}} ->
+          Hex.Shell.error(Hex.WorkloadIdentity.repository_error_message(repo, reason))
+
         {:error, :bad_repo_name} ->
           Hex.Shell.error(
             "The configured repository name for your dependency #{Hex.Utils.package_name(repo, package)} does not " <>

lib/hex/remote_converger.ex

@@ -996,37 +996,50 @@ defmodule Hex.RemoteConverger do
   # ahead of use the session; an API key would resolve here and leave the
   # session to be refreshed mid-fetch, past the point where anything can be
   # asked.
+  #
+  # Without a session, a CI job that can authenticate with a workload identity
+  # fetches with that instead, so there is nothing to ask for. It is exchanged
+  # here so the output says once that the job authenticated with it.
   @doc false
   def check_and_refresh_auth([]), do: :ok
 
-  def check_and_refresh_auth(_organizations) do
-    if Hex.State.fetch!(:offline) do
-      :ok
-    else
-      config = Hex.API.Client.config([])
+  def check_and_refresh_auth(organizations) do
+    cond do
+      Hex.State.fetch!(:offline) ->
+        :ok
 
-      case Hex.Auth.with_session_api(:read, config, fn _config -> :ok end, auth_inline: true) do
-        :ok ->
-          :ok
+      not Hex.OAuth.has_tokens?() and Hex.WorkloadIdentity.available?() ->
+        Hex.WorkloadIdentity.repository_auth(organizations)
 
-        {:error, {:auth_error, :auth_declined}} ->
-          Hex.Shell.warn(
-            "Private packages will not be available. " <>
-              "Run `mix hex.user auth` to authenticate."
-          )
-
-        {:error, {:auth_error, :token_refresh_unavailable}} ->
-          Hex.Shell.warn(
-            "Could not reach Hex to renew your authentication. " <>
-              "Private packages will not be available."
-          )
-
-        {:error, _reason} ->
-          Hex.Shell.warn(
-            "Authentication failed. Private packages will not be available. " <>
-              "Run `mix hex.user auth` to authenticate."
-          )
-      end
+      true ->
+        check_and_refresh_session()
+    end
+  end
+
+  defp check_and_refresh_session do
+    config = Hex.API.Client.config([])
+
+    case Hex.Auth.with_session_api(:read, config, fn _config -> :ok end, auth_inline: true) do
+      :ok ->
+        :ok
+
+      {:error, {:auth_error, :auth_declined}} ->
+        Hex.Shell.warn(
+          "Private packages will not be available. " <>
+            "Run `mix hex.user auth` to authenticate."
+        )
+
+      {:error, {:auth_error, :token_refresh_unavailable}} ->
+        Hex.Shell.warn(
+          "Could not reach Hex to renew your authentication. " <>
+            "Private packages will not be available."
+        )
+
+      {:error, _reason} ->
+        Hex.Shell.warn(
+          "Authentication failed. Private packages will not be available. " <>
+            "Run `mix hex.user auth` to authenticate."
+        )
     end
   end
 

lib/hex/repo.ex

@@ -245,6 +245,15 @@ defmodule Hex.Repo do
     )
   end
 
+  # Resolves the repository's credentials without making a request, so one
+  # that has to be exchanged is exchanged once, ahead of the parallel fetches
+  # that use it.
+  def resolve_auth(repo) do
+    repo_config = get_repo(repo)
+    config = build_hex_core_config(repo_config, repo)
+    :mix_hex_cli_auth.resolve_repo_auth(config)
+  end
+
   def get_public_key(repo_config) when is_map(repo_config) do
     config = build_hex_core_config(repo_config, "")
 

lib/hex/scm.ex

@@ -515,6 +515,9 @@ defmodule Hex.SCM do
 
         {:error, reason}
 
+      {:error, {:auth_error, {:workload_identity_failed, reason}}} ->
+        {:error, Hex.WorkloadIdentity.repository_error_message(repo, reason)}
+
       {:error, reason} ->
         {:error, "Request failed (#{inspect(reason)})"}
     end

lib/hex/state.ex

@@ -208,7 +208,8 @@ defmodule Hex.State do
       active_policy: {:computed, nil},
       policy_filtered_versions: {:computed, []},
       policy_locked_versions: {:computed, %{}},
-      unsatisfiable_requirements: {:computed, MapSet.new()}
+      unsatisfiable_requirements: {:computed, MapSet.new()},
+      workload_identity_tokens: {:computed, %{}}
     })
   end
 

lib/hex/workload_identity.ex

@@ -21,10 +21,50 @@ defmodule Hex.WorkloadIdentity do
         []
 
       {:error, reason} ->
-        Mix.raise("Workload Identity authentication failed, " <> describe(reason, scope))
+        Mix.raise(error_message(reason, scope))
     end
   end
 
+  @doc """
+  Whether the job can authenticate with a workload identity.
+  """
+  def available? do
+    :mix_hex_oidc.detect_provider() != :none
+  end
+
+  @doc """
+  Exchanges the CI job's OIDC token for each organization's repository token.
+
+  The fetches from the repository reuse the token, or the failure. A failure
+  is reported by each fetch the repository refuses without credentials.
+  """
+  def repository_auth(organizations) do
+    Enum.each(organizations, fn organization ->
+      case Hex.Repo.resolve_auth("hexpm:#{organization}") do
+        {:ok, _repo_key, _auth_context} ->
+          Hex.Shell.info(
+            "Authenticated to the #{organization} organization with Workload Identity"
+          )
+
+        _other ->
+          :ok
+      end
+    end)
+  end
+
+  @doc """
+  The message for a failed exchange for an organization's repository.
+  """
+  def repository_error_message("hexpm:" <> organization, reason) do
+    error_message(reason, "repository:#{organization}") <>
+      ". To fetch without Workload Identity, authenticate with " <>
+      "`mix hex.organization auth #{organization} --key KEY`"
+  end
+
+  defp error_message(reason, scope) do
+    "Workload Identity authentication failed, " <> describe(reason, scope)
+  end
+
   defp describe({:oidc_audience_failed, response}, _scope) do
     "could not fetch the OIDC audience from Hex: " <> describe_response(response)
   end

lib/mix/tasks/hex.organization.ex

@@ -17,6 +17,13 @@ defmodule Mix.Tasks.Hex.Organization do
   For CI, generate an organization key with `mix hex.organization key ORGANIZATION generate`
   and pass it with `mix hex.organization auth ORGANIZATION --key KEY`.
 
+  A GitHub Actions job with the `id-token: write` permission can fetch the
+  organization's packages without a key, using Workload Identity, sometimes also
+  known as "Trusted Publishing". Configure a workload identity with the `read`
+  or `write` role for the organization on hex.pm. When neither an organization
+  key nor an authenticated user is configured, Hex exchanges the job's OIDC
+  token for a short-lived token that can fetch the organization's packages.
+
   > #### Deprecation {: .warning}
   >
   > Authorizing an organization without `--key` is deprecated and will be removed.

lib/mix/tasks/hex.publish.ex

@@ -33,15 +33,16 @@ defmodule Mix.Tasks.Hex.Publish do
 
   ## Workload Identity
 
-  Workload Identity is sometimes also known as "Trusted Publishing". In a GitHub
-  Actions job with the `id-token: write` permission, and when no `HEX_API_KEY` or
-  other Hex credential is configured, the job's OIDC token is exchanged for a
-  short-lived token that can only publish this package. Configure a workload
-  identity on hex.pm first. A public package's workload identity is configured
-  on the package, so the package must already exist and its first release has
-  to be published with a regular account. A package in an organization's
-  repository uses the organization's workload identities, and one with the
-  `write` role can also create the package.
+  Workload Identity, sometimes also known as "Trusted Publishing", lets a GitHub
+  Actions job publish without storing an API key. When the job has the
+  `id-token: write` permission and no `HEX_API_KEY` or other Hex credential is
+  configured, `mix hex.publish` exchanges the job's OIDC token for a short-lived
+  token that can only publish this package. Configure a workload identity on
+  hex.pm first. A public package's workload identity is configured on the
+  package, so the package must already exist and its first release has to be
+  published with a regular account. A package in an organization's repository
+  uses the organization's workload identities, and one with the `write` role can
+  also create the package.
 
   ## Reverting a package
 

src/mix_hex_advisory.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Display-time deduplication of security advisories.

src/mix_hex_api.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API

src/mix_hex_api_auth.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Authentication.

src/mix_hex_api_key.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Keys.

src/mix_hex_api_oauth.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - OAuth.

src/mix_hex_api_organization.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Organizations.

src/mix_hex_api_organization_member.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Organization Members.

src/mix_hex_api_package.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Packages.

src/mix_hex_api_package_owner.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Package Owners.

src/mix_hex_api_release.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Releases.

src/mix_hex_api_short_url.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Short URLs.

src/mix_hex_api_user.erl

@@ -1,4 +1,4 @@
-%% Vendored from hex_core v0.19.0 (69f91eb), do not edit manually
+%% Vendored from hex_core v0.19.0 (6ded3af), do not edit manually
 
 %% @doc
 %% Hex HTTP API - Users.