From 61a705444c5f6c45161f43a8d5bb5f3623424445 Mon Sep 17 00:00:00 2001 From: Yorick Downe Date: Wed, 3 Jun 2026 10:52:05 +0000 Subject: [PATCH] Support Nimbus EraE import --- default.env | 4 +- ethd | 4 +- nimbus-el.yml | 2 +- nimbus-el/docker-entrypoint.sh | 94 ++++++++++++++++------------------ 4 files changed, 50 insertions(+), 54 deletions(-) diff --git a/default.env b/default.env index 7df7168b3..700f78a0b 100644 --- a/default.env +++ b/default.env @@ -265,7 +265,7 @@ RETH_SNAPSHOT= NETHERMIND_FLATDB= # EraE URL, see https://github.com/eth-clients/e2store-format-specs/ and https://ethpandaops.io/data/history/ # If this URL is provided, an EL that supports EraE import will use it when fresh syncing -ERA_URL= +ERE_URL= # If you want debug logs, set this to debug instead of info LOG_LEVEL=info @@ -548,4 +548,4 @@ DOCKER_ROOT=/var/lib/docker DOCKER_SOCK=/var/run/docker.sock # Used by ethd update - please do not adjust -ENV_VERSION=57 +ENV_VERSION=58 diff --git a/ethd b/ethd index 71bb44f48..88b4c9a31 100755 --- a/ethd +++ b/ethd @@ -1851,8 +1851,8 @@ __update_value_in_env() { __env_migrate() { - local old_vars=( ) - local new_vars=( ) + local old_vars=( ERA_URL ) + local new_vars=( ERE_URL ) local error local line local index diff --git a/nimbus-el.yml b/nimbus-el.yml index 0bf622a08..6b2bf3abf 100644 --- a/nimbus-el.yml +++ b/nimbus-el.yml @@ -27,7 +27,7 @@ services: - EL_EXTRAS=${EL_EXTRAS:-} - NODE_TYPE=${EL_NODE_TYPE:-pre-merge-expiry} - NETWORK=${NETWORK} - - ERA_URL=${ERA_URL:-} + - ERE_URL=${ERE_URL:-} volumes: - nimbus-el-data:/var/lib/nimbus - /etc/localtime:/etc/localtime:ro diff --git a/nimbus-el/docker-entrypoint.sh b/nimbus-el/docker-entrypoint.sh index fd7150fa7..1ae5076d7 100755 --- a/nimbus-el/docker-entrypoint.sh +++ b/nimbus-el/docker-entrypoint.sh @@ -19,28 +19,26 @@ __strip_empty_args() { } -__download_era_files() { -# Copyright (c) 2025 Status Research & Development GmbH. Licensed under -# either of: +__download_ere_files() { +# Copyright (c) 2025 Status Research & Development GmbH and 2026 Eth Docker maintainers. +# Licensed under either of: # - Apache License, version 2.0 # - MIT license # at your option. This file may not be copied, modified, or distributed except # according to those terms. -# Usage: __download_era_files +# Usage: __download_ere_files local download_url local download_dir local base_url - local urls_raw_file - local urls_file local completed local percent local total_files local aria_pid if [[ $# -ne 2 ]]; then - echo "__download_era_files called without . This is a bug." + echo "__download_ere_files called without . This is a bug." exit 70 fi @@ -50,62 +48,48 @@ __download_era_files() { mkdir -p "${download_dir}" cd "${download_dir}" || { echo "Could not change directory to ${download_dir}. This is a bug."; exit 70; } - # Generate safe temp files for URL lists - urls_raw_file=$(mktemp) - urls_file=$(mktemp) - - # Scrape and filter - curl -s "${download_url}" | \ - grep -Eo 'href="[^"]+"' | \ - cut -d'"' -f2 | \ - grep -Ei '\.(era|era1|txt)$' | \ - sort -u > "${urls_raw_file}" - - # Remove trailing file (like index.html) to get actual base path - base_url=$(echo "${download_url}" | sed -E 's|/[^/]*\.[a-zA-Z0-9]+$||') - - # 🔧 Normalize base URL (handle trailing slash or index.html) + # 🔧 Normalize base URL (handle trailing slash) case "${download_url}" in - */index.html) base_url="${download_url%/index.html}" ;; */) base_url="${download_url%/}" ;; *) base_url="${download_url}" ;; esac - # Prepend full URL - awk -v url="${base_url}" '{ print url "/" $0 }' "${urls_raw_file}" > "${urls_file}" - - total_files=$(wc -l < "${urls_file}") - - if [[ "${total_files}" -eq 0 ]]; then - echo "❌ No .era, .era1, or .txt files found at ${download_url}" - exit 1 - fi - - aria2c -x 8 -j 5 -c -i "${urls_file}" \ + curl -sS -O "${base_url}/urls.txt" + total_files=$(wc -l < urls.txt) + aria2c -x 8 -j 5 -c -i urls.txt \ --dir="." \ --console-log-level=warn \ --quiet=true \ --summary-interval=0 \ + --continue=true \ > /dev/null 2>&1 & aria_pid=$! - echo "Downloading Era/Era1 history files" + echo "Downloading EraE history files" echo "📥 Starting download of ${total_files} files..." while kill -0 "${aria_pid}" 2> /dev/null; do - completed=$(find . -type f \( -name '*.era' -o -name '*.era1' -o -name '*.txt' \) | wc -l) + completed=$(find . -type f \( -name '*.erae' -o -name '*.ere' \) | wc -l) percent=$(awk "BEGIN { printf \"%.1f\", (${completed}/${total_files})*100 }") echo "📦 Download Progress: ${percent}% complete (${completed} / ${total_files} files)" sleep 10 done - completed=$(find . -type f \( -name '*.era' -o -name '*.era1' -o -name '*.txt' \) | wc -l) - echo "📦 Download Progress: 100% complete (${completed} / ${total_files} files)" + wait "${aria_pid}" && exitstatus=0 || exitstatus=$? + if [[ "${exitstatus}" -ne 0 ]]; then + echo "EraE download failed with exit code ${exitstatus}" + exit "${exitstatus}" + fi - # ✅ Cleanup temp files - rm -f "${urls_raw_file}" "${urls_file}" + completed=$(find . -type f \( -name '*.erae' -o -name '*.ere' \) | wc -l) + echo "📦 Download Progress: 100% complete (${completed} / ${total_files} files)" echo "✅ All files downloaded to: ${download_dir}" + + echo "Verifying checksums" + curl -sS -O "${base_url}/checksums_sha256.txt" + sha256sum -c checksums_sha256.txt --ignore-missing + echo "✅ All checksums verified" } @@ -184,15 +168,27 @@ else fi # EraE import -# Not supported in Nimbus EL yet. Adjust parameters to ACTUAL behavior once it is -#if [[ -n "${ERA_URL}" && ! -d /var/lib/nimbus/nimbus && ! "${NETWORK}" =~ ^https?:// ]]; then # Fresh sync and named network -# __download_era_files "${ERA_URL}" /var/lib/nimbus/era - -# Word splitting is desired for the command line parameters -# shellcheck disable=SC2086 -# nimbus executionClient import --network=${NETWORK} --data-dir=/var/lib/nimbus --era-dir=/var/lib/nimbus/era -# rm -rf /var/lib/nimbus/era -#fi +if [[ -n "${ERE_URL}" && ! -f /var/lib/nimbus/ere-import-complete && ! "${NETWORK}" =~ ^https?:// ]]; then # Fresh sync and named network + if [[ "${NODE_TYPE}" =~ ^(full|archive)$ ]]; then + echo "Starting EraE history import from ${ERE_URL}" + if [[ ! -f /var/lib/nimbus/ere-download-complete ]]; then + __download_ere_files "${ERE_URL}" /var/lib/nimbus/ere + touch /var/lib/nimbus/ere-download-complete + fi + # Rename legacy erae files. This can be removed once pandaops publishes .ere + find /var/lib/nimbus/ere -type f -name '*.erae' -exec sh -c ' + for f; do + mv -- "$f" "${f%.erae}-noproofs.ere" + done + ' sh {} + + # shellcheck disable=SC2086 + nimbus import --network=${NETWORK} --data-dir=/var/lib/nimbus --ere-dir=/var/lib/nimbus/ere + touch /var/lib/nimbus/ere-import-complete + rm -rf /var/lib/nimbus/ere + else + echo "Nimbus is neither a full nor archive node, it uses ${NODE_TYPE}. Skipping EraE import." + fi +fi __strip_empty_args "$@" set -- "${__args[@]}"