From b16d4d125590e476aad0f76edffcb11435cd5f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Wed, 7 Oct 2026 15:35:03 +0200 Subject: [PATCH 1/2] Sign Docker images and record max-mode provenance The agents sign every image they push with cosign 3.1.3, using a Cloud KMS key they reach as the bob service account and find through BOB_COSIGN_KEY. `sign.sh` signs the digest of the index that `docker push` or `docker buildx imagetools create` produced, once per per-arch image and once per manifest. cosign stores the signature as an OCI referrer, which Docker Hub supports, so no tags are added to the repositories Bob reconciles, and records it in the public Rekor log. The digest is hashed from `imagetools inspect --raw` because buildx 0.17.1 ignores a `--format` template. `docker build` records provenance in max mode, which adds the build-arg values (Elixir, OTP and OS versions, which aren't secret) and the Dockerfile to the attestation BuildKit already attaches. The README documents verifying an image against cosign.pub and reading its provenance. --- Dockerfile | 2 ++ README.md | 17 +++++++++++++++++ priv/scripts/docker/elixir.sh | 4 +++- priv/scripts/docker/erlang.sh | 3 +++ priv/scripts/docker/manifest.sh | 2 ++ priv/scripts/docker/sign.sh | 20 ++++++++++++++++++++ 6 files changed, 47 insertions(+), 1 deletion(-) create mode 100755 priv/scripts/docker/sign.sh diff --git a/Dockerfile b/Dockerfile index 7e54b46d..9526d055 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,8 @@ RUN mkdir -p /usr/libexec/docker/cli-plugins && \ -o /usr/libexec/docker/cli-plugins/docker-buildx && \ chmod +x /usr/libexec/docker/cli-plugins/docker-buildx +COPY --from=ghcr.io/sigstore/cosign/cosign:v3.1.3@sha256:9e5c2f2edc34351160407ca3416c61855bdf9403c3c5936e0f0be7fc261611b8 /ko-app/cosign /usr/local/bin/cosign + RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ apt update -y && apt install --no-install-recommends -y google-cloud-cli diff --git a/README.md b/README.md index b87ba97d..9670ad59 100644 --- a/README.md +++ b/README.md @@ -80,3 +80,20 @@ Requesting an image also reserves it from cleanup, permanently. If the image alr Cleanup only touches `hexpm/elixir-amd64` and `hexpm/elixir-arm64`, which hold the single-arch images the multi-arch manifests are assembled from rather than images meant to be pulled directly. They keep the last 30 days of builds; older single-arch tags are removed, and the multi-arch image they back stays available. Tags reserved by a build request are never removed and are flagged as `reserved`. To pin a single-arch tag indefinitely, request it at https://bob.hex.pm/request. + +### Verifying images + +Images are signed with [cosign](https://github.com/sigstore/cosign) using a key held in Google Cloud KMS, and every signature is recorded in the public [Rekor](https://docs.sigstore.dev/logging/overview/) transparency log. The public key is [`cosign.pub`](cosign.pub). Verify an image with: + +``` +curl -fsSLO https://raw.githubusercontent.com/hexpm/bob/main/cosign.pub +cosign verify --key cosign.pub hexpm/elixir: +``` + +Tags built before signing was added have no signature. + +Each image also carries a [SLSA provenance](https://docs.docker.com/build/metadata/attestations/slsa-provenance/) attestation recording the build arguments, the Dockerfile and the digests of the base images it was built from: + +``` +docker buildx imagetools inspect hexpm/elixir: --format '{{json .Provenance}}' +``` diff --git a/priv/scripts/docker/elixir.sh b/priv/scripts/docker/elixir.sh index 1bf4dbd7..756e4348 100755 --- a/priv/scripts/docker/elixir.sh +++ b/priv/scripts/docker/elixir.sh @@ -11,7 +11,7 @@ arch=$5 tag=${elixir}-erlang-${erlang}-${os}-${os_version} erlang_major=$(echo "${erlang}" | awk 'match($0, /^[0-9][0-9]/) { print substr( $0, RSTART, RLENGTH )}') -docker build --ulimit nofile=1024:1024 -t hexpm/elixir-${arch}:${tag} --build-arg ELIXIR=${elixir} --build-arg ERLANG=${erlang} --build-arg ERLANG_MAJOR=${erlang_major} --build-arg OS_VERSION=${os_version} --build-arg ARCH=${arch} -f ${SCRIPT_DIR}/docker/elixir-${os}.dockerfile ${SCRIPT_DIR}/docker +docker build --ulimit nofile=1024:1024 --provenance=mode=max -t hexpm/elixir-${arch}:${tag} --build-arg ELIXIR=${elixir} --build-arg ERLANG=${erlang} --build-arg ERLANG_MAJOR=${erlang_major} --build-arg OS_VERSION=${os_version} --build-arg ARCH=${arch} -f ${SCRIPT_DIR}/docker/elixir-${os}.dockerfile ${SCRIPT_DIR}/docker # Smoke test docker run --rm hexpm/elixir-${arch}:${tag} elixir -v @@ -23,3 +23,5 @@ docker push docker.io/hexpm/elixir-${arch}:${tag} || (sleep $((20 + $RANDOM % 40)) && docker push docker.io/hexpm/elixir-${arch}:${tag}) || (sleep $((20 + $RANDOM % 40)) && docker push docker.io/hexpm/elixir-${arch}:${tag}) || (exit 1) + +${SCRIPT_DIR}/docker/sign.sh hexpm/elixir-${arch} ${tag} diff --git a/priv/scripts/docker/erlang.sh b/priv/scripts/docker/erlang.sh index f6562846..da0e4e0d 100755 --- a/priv/scripts/docker/erlang.sh +++ b/priv/scripts/docker/erlang.sh @@ -39,6 +39,7 @@ fi docker build \ --ulimit nofile=1024:1024 \ + --provenance=mode=max \ -t hexpm/erlang-${arch}:${tag} \ --build-arg ARCH=${arch} \ --build-arg ERLANG=${erlang} \ @@ -58,3 +59,5 @@ docker push docker.io/hexpm/erlang-${arch}:${tag} || (sleep $((20 + $RANDOM % 40)) && docker push docker.io/hexpm/erlang-${arch}:${tag}) || (sleep $((20 + $RANDOM % 40)) && docker push docker.io/hexpm/erlang-${arch}:${tag}) || (exit 1) + +${SCRIPT_DIR}/docker/sign.sh hexpm/erlang-${arch} ${tag} diff --git a/priv/scripts/docker/manifest.sh b/priv/scripts/docker/manifest.sh index 6b26b611..ed4635b3 100755 --- a/priv/scripts/docker/manifest.sh +++ b/priv/scripts/docker/manifest.sh @@ -20,3 +20,5 @@ docker buildx imagetools create -t hexpm/${kind}:${tag} ${arch_images} || (sleep $((10 + $RANDOM % 20)) && docker buildx imagetools create -t hexpm/${kind}:${tag} ${arch_images}) || (sleep $((10 + $RANDOM % 20)) && docker buildx imagetools create -t hexpm/${kind}:${tag} ${arch_images}) || (exit 1) + +${SCRIPT_DIR}/docker/sign.sh hexpm/${kind} ${tag} diff --git a/priv/scripts/docker/sign.sh b/priv/scripts/docker/sign.sh new file mode 100755 index 00000000..392cdced --- /dev/null +++ b/priv/scripts/docker/sign.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -euox pipefail + +repo=$1 +tag=$2 + +# Sign the digest that was pushed, since the tag can be moved +digest=sha256:$(docker buildx imagetools inspect --raw ${repo}:${tag} | sha256sum | cut -d ' ' -f 1) + +# Cloud KMS is called as the service account whose key Bob.Application writes here +export GOOGLE_APPLICATION_CREDENTIALS=/boto/keyfile.json + +# This command has a tendency to intermittently fail +cosign sign --yes --key ${BOB_COSIGN_KEY} ${repo}@${digest} || + (sleep $((20 + $RANDOM % 40)) && cosign sign --yes --key ${BOB_COSIGN_KEY} ${repo}@${digest}) || + (sleep $((20 + $RANDOM % 40)) && cosign sign --yes --key ${BOB_COSIGN_KEY} ${repo}@${digest}) || + (sleep $((20 + $RANDOM % 40)) && cosign sign --yes --key ${BOB_COSIGN_KEY} ${repo}@${digest}) || + (sleep $((20 + $RANDOM % 40)) && cosign sign --yes --key ${BOB_COSIGN_KEY} ${repo}@${digest}) || + (exit 1) From 129c0aa0aa99f2aa9a2d4b6f509ea17b694a546d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Wed, 7 Oct 2026 15:57:28 +0200 Subject: [PATCH 2/2] Add the public key images are signed with cosign.pub is version 1 of projects/hexpm-prod/locations/us-east4/keyRings/bob/cryptoKeys/cosign, created by https://github.com/hexpm/hexpm-ops/pull/386. --- cosign.pub | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 cosign.pub diff --git a/cosign.pub b/cosign.pub new file mode 100644 index 00000000..2db80db8 --- /dev/null +++ b/cosign.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEWV4rz84tD9WropTPo3OaJPGhNw7+ +4klLaSVMgijgDryNV6JBWG3ZEZ1x6qo2XPMeo9blVblN0NX1LoRQtkXP1Q== +-----END PUBLIC KEY-----