From 896e6e8ecf7395b3a36a307915128b7c743b847f Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Sun, 28 Dec 2025 21:01:24 -0500 Subject: [PATCH 1/8] Update CI actions --- .github/actions/collect_artefacts/action.yml | 2 +- .github/actions/init-python-env/action.yaml | 2 +- .github/actions/init-scala-env/action.yaml | 4 +-- .github/workflows/ci.yml | 26 ++++++++++---------- .github/workflows/docs.yml | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/actions/collect_artefacts/action.yml b/.github/actions/collect_artefacts/action.yml index 27575e34f..87c0beee7 100644 --- a/.github/actions/collect_artefacts/action.yml +++ b/.github/actions/collect_artefacts/action.yml @@ -4,7 +4,7 @@ runs: using: "composite" steps: - name: upload core dumps - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v6 with: name: core-dumps path: /tmp/core_dumps \ No newline at end of file diff --git a/.github/actions/init-python-env/action.yaml b/.github/actions/init-python-env/action.yaml index 5eeeaa5f8..32b3070b2 100644 --- a/.github/actions/init-python-env/action.yaml +++ b/.github/actions/init-python-env/action.yaml @@ -18,7 +18,7 @@ runs: steps: - name: Load cached Poetry installation id: cached-poetry - uses: actions/cache@v3 + uses: actions/cache@v5 with: path: ~/.local # the path depends on the OS, this is linux key: poetry-${{inputs.poetry_version}}-0 # increment to reset cache diff --git a/.github/actions/init-scala-env/action.yaml b/.github/actions/init-scala-env/action.yaml index 902f8de40..3d475f773 100644 --- a/.github/actions/init-scala-env/action.yaml +++ b/.github/actions/init-scala-env/action.yaml @@ -3,8 +3,8 @@ description: setup scala environment runs: using: "composite" steps: - - uses: coursier/cache-action@v6 - - uses: coursier/setup-action@v1 + - uses: coursier/cache-action@v7 + - uses: coursier/setup-action@v2 with: jvm: zulu:8.0.362 apps: sbt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e0014dde..dc2018e9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: jobs: build-scala: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -52,7 +52,7 @@ jobs: run: make build-scala - name: Cache Spark Assembly - uses: actions/cache@v3 + uses: actions/cache@v5 with: path: ./dist/* key: dist-${{ matrix.spark_version }}-${{ github.sha }} @@ -60,7 +60,7 @@ jobs: build-python: # scala/* branches are not supposed to change python code, trust them if: ${{ !startsWith(github.event.inputs.from_branch, 'scala/') }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: build-scala strategy: @@ -75,7 +75,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -88,7 +88,7 @@ jobs: shell: bash run: make lint-python - - uses: actions/cache@v3 + - uses: actions/cache@v5 with: path: ./dist/* key: dist-${{ matrix.spark_version }}-${{ github.sha }} @@ -102,7 +102,7 @@ jobs: publish-scala: name: Publish Scala Artifacts needs: [ build-scala, build-python ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 if: (github.event_name != 'pull_request') && startsWith(github.ref, 'refs/tags/v') strategy: @@ -114,7 +114,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -135,7 +135,7 @@ jobs: run: make build-scala - name: Cache Spark Assembly - uses: actions/cache@v3 + uses: actions/cache@v5 with: path: ./dist/* key: dist-${{ matrix.spark_version }}-${{ github.ref }} @@ -144,7 +144,7 @@ jobs: publish-python: name: Publish Scala Artifacts needs: [ publish-scala ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 if: (github.event_name != 'pull_request') && startsWith(github.ref, 'refs/tags/v') strategy: @@ -159,7 +159,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -168,7 +168,7 @@ jobs: python_version: ${{ matrix.python }} spark_version: ${{ matrix.spark_version }} - - uses: actions/cache@v3 + - uses: actions/cache@v5 with: path: ./dist/* key: dist-${{ matrix.spark_version }}-${{ github.ref }} @@ -182,7 +182,7 @@ jobs: # uses: ./.github/actions/upload_artefacts # TODO: Where does this go, do we need it? -# - uses: actions/cache@v3 +# - uses: actions/cache@v5 # with: # path: ./dist/* # key: dist-${{ github.sha }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ddf7b107d..03d6b9966 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: coursier/cache-action@v6 + - uses: coursier/cache-action@v7 - uses: olafurpg/setup-scala@v13 with: java-version: adopt@1.11 From 2e07b3fe0af072776e8fbb4299540ba279a9d114 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 29 Dec 2025 12:39:51 -0500 Subject: [PATCH 2/8] Install setuptools --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index d712d4064..73214aaaa 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ SPARK_VERSION ?= 3.4.0 DIST_DIR = ./dist +# poetry settings +export SETUPTOOLS_USE_DISTUTILS=stdlib + help: @echo "init - Setup the repository" @echo "clean - clean all compiled python files, build artifacts and virtual envs. Run \`make init\` anew afterwards." @@ -55,6 +58,7 @@ publish-scala: init-python: python -m venv ./.venv ./.venv/bin/python -m pip install --upgrade pip + poetry self add "setuptools<80" poetry self add "poetry-dynamic-versioning[plugin]" poetry install poetry add pyspark@${SPARK_VERSION} From 9ef3b406750366c78d92d7fcfc3a44fc33a6ac53 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 29 Dec 2025 15:34:00 -0500 Subject: [PATCH 3/8] Attempt to reinstall poetry --- .github/actions/init-python-env/action.yaml | 7 +++++++ Makefile | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/actions/init-python-env/action.yaml b/.github/actions/init-python-env/action.yaml index 32b3070b2..e75fa2c43 100644 --- a/.github/actions/init-python-env/action.yaml +++ b/.github/actions/init-python-env/action.yaml @@ -37,6 +37,13 @@ runs: python-version: ${{ inputs.python_version }} cache: 'poetry' + - name: Install Poetry (fixed) + shell: bash + run: | + pipx uninstall poetry || true + pipx install poetry + pipx inject poetry "setuptools<80" + - name: Install Poetry project dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' env: diff --git a/Makefile b/Makefile index 73214aaaa..d712d4064 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,6 @@ SPARK_VERSION ?= 3.4.0 DIST_DIR = ./dist -# poetry settings -export SETUPTOOLS_USE_DISTUTILS=stdlib - help: @echo "init - Setup the repository" @echo "clean - clean all compiled python files, build artifacts and virtual envs. Run \`make init\` anew afterwards." @@ -58,7 +55,6 @@ publish-scala: init-python: python -m venv ./.venv ./.venv/bin/python -m pip install --upgrade pip - poetry self add "setuptools<80" poetry self add "poetry-dynamic-versioning[plugin]" poetry install poetry add pyspark@${SPARK_VERSION} From 9e90141758f99edd68c6731299b46a037ae184c6 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 29 Dec 2025 15:43:50 -0500 Subject: [PATCH 4/8] Attempt to force update pip --- .github/actions/init-python-env/action.yaml | 7 ------- Makefile | 5 ++++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/actions/init-python-env/action.yaml b/.github/actions/init-python-env/action.yaml index e75fa2c43..32b3070b2 100644 --- a/.github/actions/init-python-env/action.yaml +++ b/.github/actions/init-python-env/action.yaml @@ -37,13 +37,6 @@ runs: python-version: ${{ inputs.python_version }} cache: 'poetry' - - name: Install Poetry (fixed) - shell: bash - run: | - pipx uninstall poetry || true - pipx install poetry - pipx inject poetry "setuptools<80" - - name: Install Poetry project dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' env: diff --git a/Makefile b/Makefile index d712d4064..fcebab93c 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,10 @@ publish-scala: init-python: python -m venv ./.venv - ./.venv/bin/python -m pip install --upgrade pip + ./.venv/bin/python -m pip install --upgrade pip==25.3 + ./.venv/bin/python -m pip install setuptools + poetry run python -m pip install --upgrade pip==25.3 + poetry run python -m pip install setuptools poetry self add "poetry-dynamic-versioning[plugin]" poetry install poetry add pyspark@${SPARK_VERSION} From f4ff0d0fbae3ca841f09892782bf376270bdd354 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 29 Dec 2025 15:51:23 -0500 Subject: [PATCH 5/8] Rm gloabl pip update --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index fcebab93c..57f28e44d 100644 --- a/Makefile +++ b/Makefile @@ -54,9 +54,6 @@ publish-scala: init-python: python -m venv ./.venv - ./.venv/bin/python -m pip install --upgrade pip==25.3 - ./.venv/bin/python -m pip install setuptools - poetry run python -m pip install --upgrade pip==25.3 poetry run python -m pip install setuptools poetry self add "poetry-dynamic-versioning[plugin]" poetry install From a98e5a3dd446f75056fe9b9fac35e0bd10e34883 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 29 Dec 2025 21:04:40 -0500 Subject: [PATCH 6/8] Install ubuntu py3 distutils --- .github/actions/init-python-env/action.yaml | 6 ++++++ Makefile | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/init-python-env/action.yaml b/.github/actions/init-python-env/action.yaml index 32b3070b2..de4a3626d 100644 --- a/.github/actions/init-python-env/action.yaml +++ b/.github/actions/init-python-env/action.yaml @@ -37,6 +37,12 @@ runs: python-version: ${{ inputs.python_version }} cache: 'poetry' + - name: Install distutils + shell: bash + run: | + sudo apt-get update + sudo apt-get install python${{ inputs.python_version }}-distutils -y + - name: Install Poetry project dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' env: diff --git a/Makefile b/Makefile index 57f28e44d..f79fbd00a 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ publish-scala: init-python: python -m venv ./.venv - poetry run python -m pip install setuptools + poetry run python -m pip install --upgrade setuptools poetry self add "poetry-dynamic-versioning[plugin]" poetry install poetry add pyspark@${SPARK_VERSION} From 0249c6659752426f1bf65c05032ea34df1dc9b93 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 29 Dec 2025 21:31:16 -0500 Subject: [PATCH 7/8] downgrade ubuntu --- .github/actions/init-python-env/action.yaml | 2 +- .github/workflows/ci.yml | 8 ++++---- .github/workflows/docs.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/init-python-env/action.yaml b/.github/actions/init-python-env/action.yaml index de4a3626d..856690133 100644 --- a/.github/actions/init-python-env/action.yaml +++ b/.github/actions/init-python-env/action.yaml @@ -41,7 +41,7 @@ runs: shell: bash run: | sudo apt-get update - sudo apt-get install python${{ inputs.python_version }}-distutils -y + sudo apt-get install python3-setuptools -y - name: Install Poetry project dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc2018e9f..904b0761b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: jobs: build-scala: - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 strategy: matrix: @@ -60,7 +60,7 @@ jobs: build-python: # scala/* branches are not supposed to change python code, trust them if: ${{ !startsWith(github.event.inputs.from_branch, 'scala/') }} - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 needs: build-scala strategy: @@ -102,7 +102,7 @@ jobs: publish-scala: name: Publish Scala Artifacts needs: [ build-scala, build-python ] - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 if: (github.event_name != 'pull_request') && startsWith(github.ref, 'refs/tags/v') strategy: @@ -144,7 +144,7 @@ jobs: publish-python: name: Publish Scala Artifacts needs: [ publish-scala ] - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 if: (github.event_name != 'pull_request') && startsWith(github.ref, 'refs/tags/v') strategy: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 03d6b9966..ca3a6bd8f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,7 +13,7 @@ on: jobs: docs: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 container: image: s22s/debian-openjdk-conda-gdal:6790f8d From aed5d33f4b495e1ebf99068a57c515fe615ac98f Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 29 Dec 2025 21:43:55 -0500 Subject: [PATCH 8/8] cleanup --- .github/actions/init-python-env/action.yaml | 6 ------ Makefile | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/actions/init-python-env/action.yaml b/.github/actions/init-python-env/action.yaml index 856690133..32b3070b2 100644 --- a/.github/actions/init-python-env/action.yaml +++ b/.github/actions/init-python-env/action.yaml @@ -37,12 +37,6 @@ runs: python-version: ${{ inputs.python_version }} cache: 'poetry' - - name: Install distutils - shell: bash - run: | - sudo apt-get update - sudo apt-get install python3-setuptools -y - - name: Install Poetry project dependencies # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' env: diff --git a/Makefile b/Makefile index f79fbd00a..d712d4064 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ publish-scala: init-python: python -m venv ./.venv - poetry run python -m pip install --upgrade setuptools + ./.venv/bin/python -m pip install --upgrade pip poetry self add "poetry-dynamic-versioning[plugin]" poetry install poetry add pyspark@${SPARK_VERSION}