Fixup logging, fmt, linting

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: |
+          sudo apt install --quiet shellcheck
+          python -m pip install --upgrade pip
+          pip install -r requirements.lock
+          cp env-sample .env
+
+      - name: Run lint script
+        run: scripts/lint.sh
+

.gitignore

@@ -394,5 +394,8 @@ webhooksecrets.csv
 # Do not store user mappings file
 user_conflicts*
 
+# Ignore gitlab output files
+gitlab_size_analysis*.csv
+
 # Ignore archive files
 *.tar.gz

README.md

@@ -14,12 +14,12 @@ some custom tooling helps fill the gaps that both these tools leave in migrating
 
 ## Development
 
-To work on this app, you need a Macbook, Linux or Windows system.
+To work on this app, you need a macOS, Linux or Windows system.
 
 The scripts are Python 3 scripts, in the `scripts` directory. They require some environment variables to be set for 
 them to work, see each script for details.
 
-This file has some Git LFS files, see 
+This repository has some Git LFS files, see 
 [GitHub's documentation on working with large files](https://docs.github.com/en/repositories/working-with-files/managing-large-files/configuring-git-large-file-storage)
 for how to enable LFS on your workstation.
 
@@ -28,6 +28,17 @@ then source it with:
 
     set -o allexport && source .env && set +o allexport
 
+To develop the python scripts you will need Python 3.9+ installed.
+
+To install the dependencies, you can create a virtual environment and install the dependencies like this:
+```bash
+rm -rf .venv
+python3 -m venv .venv
+source .venv/bin/activate
+pip install --upgrade pip
+pip install -r requirements.lock
+```
+
 ### Git LFS
 
 This repository requires Git LFS to be installed on any system that clones the repository.  See 
@@ -84,6 +95,8 @@ repoList file in `REPO_LIST`:
 
 ### Requirements
 
+This uses pip to manage Python dependencies.
+
 -   **requirements.txt**
     All first level dependencies should be declared in `requirements.txt`.
 -   **requirements-test.txt**

env-sample

@@ -2,7 +2,7 @@
 # env.sample
 #
 # Copy this to .env and customize it with real access tokens.
-# 
+#
 # Then source it before running commands in the scripts:
 #
 #    set -o allexport && source .env && set +o allexport
@@ -13,7 +13,7 @@
 # These rest of the scripts use the same conventions as [github/gh-gei](https://github.com/github/gh-gei) for GitHub tokens. These are:
 #
 #    GH_PAT         Personal access token for github.com (migration destination)
-#    GH_SOURCE_PAT  Personal access token for github.example.com (migration source) 
+#    GH_SOURCE_PAT  Personal access token for github.example.com (migration source)
 #
 # These tokens must have the scopes that are defined in the `github/gh-gei` documentation, see:
 #   https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer#required-scopes-for-personal-access-tokens
@@ -36,3 +36,6 @@ BUILDKITE_TOKEN=redacted
 # For testing, set this with scripts/getVaultToken.sh:
 #  export VAULT_TOKEN=$(scripts/getVaultToken.sh)
 #VAULT_TOKEN=redacted
+
+# Optional: if you want to use `scripts/gitlab-export.py` set this:
+GITLAB_TOKEN=redacted

requirements.lock

@@ -1,76 +1,83 @@
-aiohttp==3.9.5
-aiosignal==1.3.1
-asttokens==2.4.1
-attrs==23.2.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.13
+aiosignal==1.3.2
+asttokens==3.0.0
+async-timeout==5.0.1
+attrs==25.3.0
 bandit==1.7.8
 black==24.4.1
-certifi==2024.2.2
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-cryptography==42.0.5
-decorator==5.1.1
-Deprecated==1.2.14
-et-xmlfile==1.1.0
-executing==2.0.1
-fancycompleter==0.9.1
+certifi==2025.6.15
+cffi==1.17.1
+charset-normalizer==3.4.2
+click==8.1.8
+cryptography==45.0.4
+decorator==5.2.1
+Deprecated==1.2.18
+et_xmlfile==2.0.0
+exceptiongroup==1.3.0
+executing==2.2.0
+fancycompleter==0.11.1
 flake8==7.0.0
 flake8-bugbear==24.4.21
 flake8-comprehensions==3.14.0
 flake8-print==5.0.0
-frozenlist==1.4.1
-hvac==2.2.0
-idna==3.7
+frozenlist==1.7.0
+hvac==2.3.0
+idna==3.10
 IPy==1.1
 ipython==8.18.1
 isort==5.13.2
-jedi==0.19.1
+jedi==0.19.2
 markdown-it-py==3.0.0
 matplotlib-inline==0.1.7
 mccabe==0.7.0
 mdurl==0.1.2
-multidict==6.0.5
+multidict==6.5.0
 mypy==1.9.0
-mypy-extensions==1.0.0
-numpy==1.26.4
-openpyxl==3.1.2
-packaging==24.0
-pandas==2.2.2
+mypy_extensions==1.1.0
+numpy==2.0.2
+openpyxl==3.1.5
+packaging==25.0
+pandas==2.3.0
 parso==0.8.4
 pathspec==0.12.1
-pbr==6.0.0
-pdbpp==0.10.3
+pbr==6.1.1
+pdbpp==0.11.6
 pexpect==4.9.0
-platformdirs==4.2.1
-prompt-toolkit==3.0.43
+platformdirs==4.3.8
+prompt_toolkit==3.0.51
+propcache==0.3.2
 ptyprocess==0.7.0
-pure-eval==0.2.2
+pure_eval==0.2.3
 pycodestyle==2.11.1
 pycparser==2.22
 pyflakes==3.2.0
-PyGithub==2.3.0
-Pygments==2.17.2
-PyJWT==2.8.0
+PyGithub==2.6.1
+Pygments==2.19.1
+PyJWT==2.10.1
 PyNaCl==1.5.0
-pyrepl==0.9.0
+pyrepl==0.11.3.post1
 python-dateutil==2.9.0.post0
+python-gitlab==6.0.0
 python-graphql-client==0.4.3
-pytz==2024.1
-PyYAML==6.0.1
-requests==2.31.0
-rich==13.7.1
-six==1.16.0
+pytz==2025.2
+PyYAML==6.0.2
+requests==2.32.4
+requests-toolbelt==1.0.0
+rich==14.0.0
+six==1.17.0
 stack-data==0.6.3
-stevedore==5.2.0
+stevedore==5.4.1
+tomli==2.2.1
 traitlets==5.14.3
 types-click==7.1.8
 types-PyYAML==6.0.12.20240311
 types-requests==2.31.0.20240406
-typing_extensions==4.11.0
-tzdata==2024.1
-urllib3==2.2.1
+typing_extensions==4.14.0
+tzdata==2025.2
+urllib3==2.5.0
 wcwidth==0.2.13
-websockets==12.0
+websockets==15.0.1
 wmctrl==0.5
-wrapt==1.16.0
-yarl==1.9.4
+wrapt==1.17.2
+yarl==1.20.1

requirements.txt

@@ -4,6 +4,7 @@ PyGithub
 hvac
 openpyxl
 pandas
+python-gitlab
 python_graphql_client
 pyyaml
 requests

scripts/getWebhookList.py

@@ -12,7 +12,13 @@
 from urllib.parse import urlparse
 
 import requests
-from utils import DEFAULT_TIMEOUT, assertGetenv, getLogger, ghRateLimitSleep, DEFAULT_ORG
+from utils import (
+    DEFAULT_ORG,
+    DEFAULT_TIMEOUT,
+    assertGetenv,
+    getLogger,
+    ghRateLimitSleep,
+)
 
 token = assertGetenv(
     "GH_SOURCE_PAT", "Provide a personal access token from the source GHES instance"