diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 873a474..b627942 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -1,6 +1,7 @@ +--- name: PR checks -on: +"on": pull_request: branches: [main] types: @@ -12,48 +13,54 @@ on: - "pybdl/**" - "tests/**" - "pyproject.toml" - - ".github/workflows/pr-checks.yml" + - ".github/workflows/pr_checks.yml" concurrency: group: pr-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -permissions: - contents: read +permissions: {} jobs: conventional-pr: name: Conventional PR title runs-on: ubuntu-latest permissions: - pull-requests: read + pull-requests: read # action-semantic-pull-request: read PR title steps: - name: Check PR title - uses: amannn/action-semantic-pull-request@v5 + uses: amannn/action-semantic-pull-request@e32d7e603df1aa1ba07e981f2a23455dee596825 # v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} lint: name: Linting (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 with: enable-cache: true + cache-dependency-glob: "uv.lock" - name: Set up Python ${{ matrix.python-version }} - run: uv python install ${{ matrix.python-version }} + env: + UV_PYTHON_VERSION: ${{ matrix.python-version }} + run: uv python install "${UV_PYTHON_VERSION}" - - name: Install project (dev) - run: uv sync --dev + - name: Install project (dev + extras) + run: uv sync --all-extras --dev - name: Ruff linter run: uv run ruff check --config pyproject.toml --output-format=github pybdl/ tests/ @@ -65,22 +72,33 @@ jobs: run: uv run bandit -c pyproject.toml -r pybdl/ mypy: - name: Mypy (Python 3.12) + name: Mypy (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 with: enable-cache: true + cache-dependency-glob: "uv.lock" - - name: Set up Python 3.12 - run: uv python install 3.12 + - name: Set up Python ${{ matrix.python-version }} + env: + UV_PYTHON_VERSION: ${{ matrix.python-version }} + run: uv python install "${UV_PYTHON_VERSION}" - - name: Install project (dev) - run: uv sync --dev + - name: Install project (dev + extras) + run: uv sync --all-extras --dev - name: Mypy run: uv run mypy pybdl/ tests/ @@ -95,32 +113,37 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] permissions: contents: read - checks: write - pull-requests: write + checks: write # publish-unit-test-result-action: create check runs + pull-requests: write # publish-unit-test-result-action: annotate PR steps: - name: Checkout repo - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 with: enable-cache: true + cache-dependency-glob: "uv.lock" - name: Set up Python ${{ matrix.python-version }} - run: uv python install ${{ matrix.python-version }} + env: + UV_PYTHON_VERSION: ${{ matrix.python-version }} + run: uv python install "${UV_PYTHON_VERSION}" - - name: Install project (dev) - run: uv sync --dev + - name: Install project (dev + extras) + run: uv sync --all-extras --dev - name: Run tests with coverage run: uv run coverage run -m pytest --junitxml=./test_results.xml - name: Publish test results to GitHub - if: always() - uses: EnricoMi/publish-unit-test-result-action@v2 + if: ${{ !cancelled() }} + uses: EnricoMi/publish-unit-test-result-action@2ceeed2b8f26e15b06c3846719f75354febd9e7b # v2 with: files: ./test_results.xml fail_on: nothing diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5c05bad..8037b80 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,10 @@ +--- name: Release -on: +"on": workflow_dispatch: -permissions: - contents: write - pull-requests: write - id-token: write +permissions: {} concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -16,43 +14,47 @@ jobs: quality-checks: name: Quality checks (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] - + python-version: ["3.11", "3.12", "3.13", "3.14"] + steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 with: - enable-cache: true - cache-dependency-glob: "uv.lock" + enable-cache: false - name: Set up Python ${{ matrix.python-version }} - run: uv python install ${{ matrix.python-version }} + env: + UV_PYTHON_VERSION: ${{ matrix.python-version }} + run: uv python install "${UV_PYTHON_VERSION}" - name: Install dependencies run: uv sync --all-extras --dev - name: Run Ruff linter - if: always() + if: ${{ !cancelled() }} run: uv run ruff check --config pyproject.toml --output-format=github pybdl/ tests/ - name: Run Ruff formatter - if: always() + if: ${{ !cancelled() }} run: uv run ruff format --config pyproject.toml --check pybdl/ tests/ - name: Run Bandit security checks - if: always() + if: ${{ !cancelled() }} run: uv run bandit -c pyproject.toml -r pybdl/ - name: Run Mypy type checker - if: always() + if: ${{ !cancelled() }} run: uv run mypy pybdl/ tests/ test: @@ -61,25 +63,29 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] - + python-version: ["3.11", "3.12", "3.13", "3.14"] + permissions: contents: read - checks: write - pull-requests: write - + checks: write # publish-unit-test-result-action: create check runs + pull-requests: write # publish-unit-test-result-action: annotate PR + id-token: write # codecov-action: OIDC token when not using CODECOV_TOKEN + steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 with: - enable-cache: true - cache-dependency-glob: "uv.lock" + enable-cache: false - name: Set up Python ${{ matrix.python-version }} - run: uv python install ${{ matrix.python-version }} + env: + UV_PYTHON_VERSION: ${{ matrix.python-version }} + run: uv python install "${UV_PYTHON_VERSION}" - name: Install dependencies run: uv sync --all-extras --dev @@ -88,23 +94,23 @@ jobs: run: uv run coverage run -m pytest --junitxml=./test_results.xml - name: Generate coverage report - if: always() + if: ${{ !cancelled() }} run: | uv run coverage report uv run coverage xml - name: Publish test results - uses: EnricoMi/publish-unit-test-result-action@v2 - if: always() + uses: EnricoMi/publish-unit-test-result-action@2ceeed2b8f26e15b06c3846719f75354febd9e7b # v2 + if: ${{ !cancelled() }} with: files: ./test_results.xml check_name: Test Results (Python ${{ matrix.python-version }}) - name: Upload coverage to Codecov if: matrix.python-version == '3.12' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 with: - file: ./coverage.xml + files: ./coverage.xml flags: unittests name: codecov-umbrella fail_ci_if_error: false @@ -113,28 +119,36 @@ jobs: name: Semantic release runs-on: ubuntu-latest needs: [quality-checks, test] - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - + if: github.ref == 'refs/heads/main' + + environment: release + + permissions: + contents: write # semantic-release: tag, commit version bumps, push to main + outputs: released: ${{ steps.release.outputs.released }} version: ${{ steps.release.outputs.version }} tag: ${{ steps.release.outputs.tag }} - + steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 + persist-credentials: true # zizmor: ignore[artipacked] - name: Generate GitHub App Token id: app-token - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 with: app-id: ${{ vars.GH_APP_RW_ACCESS_APP_ID }} private-key: ${{ secrets.GH_APP_RW_ACCESS_PRIVATE_KEY }} - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 + with: + enable-cache: false - name: Set up Python run: uv python install @@ -149,7 +163,7 @@ jobs: - name: Python Semantic Release id: release - uses: python-semantic-release/python-semantic-release@v10.5.2 + uses: python-semantic-release/python-semantic-release@fb90716c1b3afbd1d88f48d7f88ba7365914a86e # v10.5.2 with: github_token: ${{ steps.app-token.outputs.token }} root_options: "-vv" @@ -159,16 +173,22 @@ jobs: runs-on: ubuntu-latest needs: release if: needs.release.outputs.released == 'true' - + + permissions: + contents: read + steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: ref: main fetch-depth: 0 + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 + with: + enable-cache: false - name: Set up Python run: uv python install @@ -177,7 +197,7 @@ jobs: run: uv build - name: Store distribution packages - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 with: name: python-package-distributions path: dist/ @@ -188,43 +208,45 @@ jobs: runs-on: ubuntu-latest needs: [release, build] if: needs.release.outputs.released == 'true' - + environment: name: pypi url: https://pypi.org/p/pybdl - + permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing - + id-token: write # Trusted publishing to PyPI + steps: - name: Download distributions - uses: actions/download-artifact@v5 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 with: name: python-package-distributions path: dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 publish-github: name: Upload to GitHub Release runs-on: ubuntu-latest needs: [release, build] if: needs.release.outputs.released == 'true' - + + environment: release + permissions: - contents: write - id-token: write - + contents: write # gh release upload: attach assets to GitHub Release + id-token: write # sigstore/gh-action-sigstore-python: OIDC for signing + steps: - name: Download distributions - uses: actions/download-artifact@v5 + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5 with: name: python-package-distributions path: dist/ - name: Sign distributions with Sigstore - uses: sigstore/gh-action-sigstore-python@v3.0.0 + uses: sigstore/gh-action-sigstore-python@f514d46b907ebcd5bedc05145c03b69c1edd8b46 # v3.0.0 with: inputs: >- ./dist/*.tar.gz @@ -232,7 +254,7 @@ jobs: - name: Generate GitHub App Token id: app-token - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 with: app-id: ${{ vars.GH_APP_RW_ACCESS_APP_ID }} private-key: ${{ secrets.GH_APP_RW_ACCESS_PRIVATE_KEY }} @@ -240,7 +262,10 @@ jobs: - name: Upload to GitHub Release env: GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} - run: >- - gh release upload - '${{ needs.release.outputs.tag }}' dist/** - --repo '${{ github.repository }}' + RELEASE_TAG: ${{ needs.release.outputs.tag }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + shopt -s nullglob + assets=(dist/*.tar.gz dist/*.whl) + gh release upload "${RELEASE_TAG}" "${assets[@]}" --repo "${GITHUB_REPOSITORY}" diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 7c934bc..be8113a 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -1,24 +1,27 @@ +--- name: Docs -on: +"on": push: branches: [main] paths: - "docs/**" - "pybdl/**" - - ".github/workflows/docs.yml" + - ".github/workflows/sphinx.yml" - "pyproject.toml" pull_request: paths: - "docs/**" - "pybdl/**" - - ".github/workflows/docs.yml" + - ".github/workflows/sphinx.yml" - "pyproject.toml" concurrency: group: docs-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: build: name: Build Sphinx docs @@ -26,14 +29,16 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7 with: enable-cache: true + cache-dependency-glob: "uv.lock" - name: Set up Python run: uv python install @@ -46,7 +51,7 @@ jobs: uv run sphinx-build -W --keep-going -b html docs docs/_build/html - name: Upload docs as artifact (PRs & pushes) - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 with: name: html-docs path: docs/_build/html @@ -54,7 +59,7 @@ jobs: - name: Upload GitHub Pages artifact (only main) if: github.event_name == 'push' && github.ref == 'refs/heads/main' - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 with: path: docs/_build/html @@ -67,9 +72,9 @@ jobs: name: github-pages url: ${{ steps.deployment.outputs.page_url }} permissions: - pages: write - id-token: write + pages: write # deploy-pages: publish to GitHub Pages + id-token: write # deploy-pages: OIDC authentication with Pages steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4915416..162ad36 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,252 @@ +--- +default_language_version: + python: python3.11 + +default_stages: + - pre-commit + +default_install_hook_types: + - pre-commit + - commit-msg + - pre-push + - post-checkout + - post-merge + +fail_fast: true + repos: + ############################################################################## + # Policy / commit message quality + ############################################################################## + - repo: https://github.com/compilerla/conventional-pre-commit + rev: 91ab4bf57e58b32adf1a122681f6ebe164d081c8 # frozen: v4.4.0-pre1 + hooks: + - id: conventional-pre-commit + name: "<> Commit" + alias: commit + stages: [commit-msg] + args: + - "--strict" + verbose: true + require_serial: true + always_run: true + + ############################################################################## + # Core formatting / hygiene + ############################################################################## + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 + hooks: + - id: trailing-whitespace + name: "<< FIX >> Trailing whitespace" + args: [--markdown-linebreak-ext=md] + - id: check-added-large-files + name: "<> Added large files" + args: [--maxkb=1024] + stages: [pre-commit] + - id: check-case-conflict + name: "<> Conflicting upper/lower case filenames" + - id: check-executables-have-shebangs + name: "<> Executables have shebangs" + stages: [pre-commit] + - id: check-illegal-windows-names + name: "<> Illegal Windows filenames" + - id: check-json + name: "<> Valid JSON" + - id: check-merge-conflict + name: "<> Merge conflict markers" + - id: check-shebang-scripts-are-executable + name: "<> Shebang scripts are executable" + - id: check-symlinks + name: "<> Valid symlinks" + - id: check-toml + name: "<> Valid TOML" + - id: check-yaml + name: "<> Valid YAML" + args: [--unsafe] + - id: destroyed-symlinks + name: "<> Destroyed symlinks" + - id: detect-private-key + name: "<> Private keys" + - id: end-of-file-fixer + name: "<< FIX >> End of file" + stages: [pre-commit] + - id: mixed-line-ending + name: "<< FIX >> Mixed line endings" + args: [--fix=auto] + - id: debug-statements + name: "<> Debug statements" + + - repo: https://github.com/codespell-project/codespell + rev: 2ccb47ff45ad361a21071a7eedda4c37e6ae8c5a # frozen: v2.4.2 + hooks: + - id: codespell + alias: fix-code + name: "<< FIX >> Code spelling" + args: [--write-changes] + exclude: ^tests/integration/access/samples/raw/ + verbose: true + + ############################################################################## + # Python lint, format, typing + ############################################################################## + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: aca6d4c8045a504e2812ea4bedff1d0a09e437bc # frozen: v0.15.8 + hooks: + - id: ruff-check + name: "<> Ruff" + args: [--fix] + - id: ruff-format + name: "<< FIX >> Ruff format" + + - repo: https://github.com/asottile/pyupgrade + rev: 75992aaa40730136014f34227e0135f63fc951b4 # frozen: v3.21.2 + hooks: + - id: pyupgrade + name: "<< FIX >> Pyupgrade" + args: [--py311-plus] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: a66e98df7b4aeeb3724184b332785976d062b92e # frozen: v1.19.1 + hooks: + - id: mypy + name: "<> mypy" + additional_dependencies: + - pydantic + - types-PyYAML + + ############################################################################## + # Security & static analysis + ############################################################################## + - repo: https://github.com/semgrep/semgrep + rev: ab584982f6ecdaaa7954a14e5350a70c060e097f # frozen: v1.156.0 + hooks: + - id: semgrep + alias: check-security + name: "<> Security (Semgrep)" + language_version: python3.11 + args: + - "--disable-version-check" + - "--quiet" + - "--skip-unknown-extensions" + - "--config=p/python" + - "--config=p/secrets" + - "--config=p/security-audit" + - "--config=p/ci" + - "--config=p/github-actions" + - "--config=p/gitleaks" + - "--error" + - "--metrics=off" + - "--severity=INFO" + - "--strict" + verbose: true + + - repo: https://github.com/trufflesecurity/trufflehog + rev: 586f66d7886cd0b037c7c245d4a6e34ef357ab10 # frozen: v3.94.1 + hooks: + - id: trufflehog + alias: secrets + name: "<> Secrets (trufflehog)" + verbose: true + + - repo: https://github.com/PyCQA/bandit + rev: 92ae8b82fb422a639f0ed8d99e96cea769594e08 # frozen: 1.9.4 + hooks: + - id: bandit + name: "<> Bandit" + args: + - -c + - pyproject.toml + - --exclude + - README.md,**/*.md,**/*.rst + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: ea2eb407b4cbce87cf0d502f36578950494f5ac9 # frozen: v1.23.1 + hooks: + - id: zizmor + name: "<> GitHub Actions (zizmor)" + args: + - "--no-progress" + - "--persona" + - "auditor" + - "--verbose" + require_serial: true + + ############################################################################## + # YAML / JSON Schema / Markdown quality + ############################################################################## + - repo: https://github.com/adrienverge/yamllint + rev: cba56bcde1fdd01c1deb3f945e69764c291a6530 # v1.38.0 + hooks: + - id: yamllint + alias: yaml + name: "<> YAML style" + verbose: true + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 13614ab716a3113145f1294ed259d9fbe5678ff3 # frozen: 0.37.1 + hooks: + - id: check-github-actions + name: "<> GitHub Actions JSON Schema" + args: ["--verbose"] + - id: check-github-workflows + name: "<> GitHub Workflows JSON Schema" + args: ["--verbose"] + + - repo: https://github.com/jackdewinter/pymarkdown + rev: 6d3c1b70d44ea2202b23552af93aca3b5017ee83 # frozen: v0.9.36 + hooks: + - id: pymarkdown + alias: fix-markdown + name: "<< FIX >> Markdown" + verbose: true + require_serial: true + entry: pymarkdown --disable-rules md007,md028,md033,md034,md041,md042 + args: ["fix"] + types: [markdown] + additional_dependencies: + - application-properties + - id: pymarkdown + alias: check-markdown + name: "<> Markdown" + verbose: true + require_serial: true + entry: > + pymarkdown + --disable-rules md007,md028,md033,md034,md041,md042 + --set 'plugins.line-length.line_length=$#120' + args: ["scan"] + types: [markdown] + additional_dependencies: + - application-properties + + ############################################################################## + # uv integration + ############################################################################## + # Keep uv.lock in sync - repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.6.5 + rev: 20d6e6698b00aed95d4f269a6309b11c2c57b202 # frozen: 0.11.2 hooks: - id: uv-lock + name: "<< FIX >> uv lock" + + ############################################################################## + # System / self-management hooks + ############################################################################## + - repo: local + hooks: + - id: prek-config + verbose: true + name: "<> prek config" + entry: prek validate-config .pre-commit-config.yaml + files: "^.pre-commit-config.yaml$" + language: system + + - id: prek-autoupdate + verbose: true + name: "<< FIX >> prek autoupdate (frozen)" + entry: prek auto-update --freeze + pass_filenames: false + language: system + require_serial: true + always_run: true diff --git a/.vscode/git-commit-instructions.md b/.vscode/git-commit-instructions.md deleted file mode 100644 index 9c8b1c7..0000000 --- a/.vscode/git-commit-instructions.md +++ /dev/null @@ -1,18 +0,0 @@ -Generate a concise, Conventional Commit message for a Python library. - -Follow these rules: -Use one of these types: chore, docs, feat, fix, refactor, style, test, revert, perf, ci, merge. -Use a short, lowercase imperative summary (max 72 chars) after type(scope):. -Scope is optional but can specify a module or feature. -Use a blank line before the detailed body (if needed). -Body explains what, why, and how (if relevant). -Use present tense and avoid personal pronouns. -Reference issues or PRs if relevant (e.g., “Closes #123”). -For breaking changes, add a BREAKING CHANGE: section. -Do not use WIP in commit messages. - -Examples: -feat(core): add async support to LDB client -fix(visualization): correct axis label formatting -docs: update usage section in README -refactor: simplify config loading logic diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index e87212a..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "github.copilot.chat.codesearch.enabled": true, - "github.copilot.chat.agent.thinkingTool": true, - "github.copilot.chat.commitMessageGeneration.instructions": [ - { - "file": "./git-commit-instructions.md" - } - ] -} \ No newline at end of file diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..2ae70bc --- /dev/null +++ b/.yamllint @@ -0,0 +1,7 @@ +--- +extends: default + +rules: + line-length: + max: 120 + level: error diff --git a/CHANGELOG.md b/CHANGELOG.md index 60c3d8e..1321d38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,4 @@ and the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0. ## [Unreleased] -*(No changes yet — this section is populated automatically on release.)* - diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep index ddeb93a..5371455 100644 --- a/docs/_static/.gitkeep +++ b/docs/_static/.gitkeep @@ -1,5 +1,2 @@ # This file ensures the _static directory is tracked in git # Sphinx will use this directory for custom static files if needed - - - diff --git a/docs/access_layer.rst b/docs/access_layer.rst index d9309d9..2e93d88 100644 --- a/docs/access_layer.rst +++ b/docs/access_layer.rst @@ -50,7 +50,7 @@ All access layer methods return pandas DataFrames, making data immediately ready # Get variables as DataFrame variables_df = bdl.variables.list_variables() - + # Use pandas operations directly filtered = variables_df[variables_df['name'].str.contains('population', case=False)] sorted_vars = variables_df.sort_values('name') @@ -142,10 +142,10 @@ Administrative unit aggregation levels (country, voivodeship, county, municipali # List all levels levels_df = bdl.levels.list_levels() - + # Get specific level level_df = bdl.levels.get_level(1) # Level 1 = country - + # Get metadata metadata_df = bdl.levels.get_levels_metadata() @@ -158,13 +158,13 @@ Subject categories and hierarchy: # List all top-level subjects subjects_df = bdl.subjects.list_subjects() - + # Get subjects under a parent child_subjects = bdl.subjects.list_subjects(parent_id="P0001") - + # Search subjects results = bdl.subjects.search_subjects(name="population") - + # Get specific subject subject_df = bdl.subjects.get_subject("P0001") @@ -177,16 +177,16 @@ Statistical variables (indicators): # List all variables variables_df = bdl.variables.list_variables() - + # Filter variables filtered = bdl.variables.list_variables( category_id="P0001", name="population" ) - + # Search variables results = bdl.variables.search_variables(name="unemployment") - + # Get specific variable variable_df = bdl.variables.get_variable("3643") @@ -203,35 +203,35 @@ Statistical data retrieval: years=[2021], unit_level=2 # Voivodeship level ) - + # Get data for multiple years df = bdl.data.get_data_by_variable( variable_id="3643", years=[2020, 2021, 2022], unit_level=2 ) - + # Get data with aggregate filter df = bdl.data.get_data_by_variable( variable_id="3643", years=[2021], aggregate_id=1 ) - + # Get data by administrative unit df = bdl.data.get_data_by_unit( unit_id="020000000000", variable_ids=["3643"], years=[2021] ) - + # Get data for a locality df = bdl.data.get_data_by_variable_locality( variable_id="3643", unit_parent_id="1465011", years=[2021] ) - + # Get data by unit locality df = bdl.data.get_data_by_unit_locality( unit_id="1465011", @@ -281,19 +281,19 @@ Administrative units (regions, cities, etc.): # List units by level voivodeships = bdl.units.list_units(level=2) # Level 2 = voivodeship - + # Search units warsaw = bdl.units.search_units(name="Warsaw") - + # Get specific unit unit_df = bdl.units.get_unit("020000000000") - + # List localities (statistical localities) localities = bdl.units.list_localities(level=6) # Level 6 = municipality - + # Search localities warsaw_localities = bdl.units.search_localities(name="Warsaw", level=6) - + # Get specific locality locality_df = bdl.units.get_locality("1465011") @@ -306,7 +306,7 @@ Data attributes (dimensions): # List all attributes attributes_df = bdl.attributes.list_attributes() - + # Get specific attribute attr_df = bdl.attributes.get_attribute("1") @@ -319,7 +319,7 @@ Measure units: # List all measures measures_df = bdl.measures.list_measures() - + # Get specific measure measure_df = bdl.measures.get_measure(1) @@ -332,7 +332,7 @@ Aggregation types: # List all aggregates aggregates_df = bdl.aggregates.list_aggregates() - + # Get specific aggregate aggregate_df = bdl.aggregates.get_aggregate("1") @@ -345,7 +345,7 @@ Available years for data: # List all available years years_df = bdl.years.list_years() - + # Get specific year metadata year_df = bdl.years.get_year(2021) @@ -430,10 +430,10 @@ Most list methods support pagination: # Fetch all pages (default, max_pages=None) all_data = bdl.variables.list_variables() - + # Fetch only first page first_page = bdl.variables.list_variables(max_pages=1, page_size=50) - + # Limit number of pages limited = bdl.variables.list_variables(max_pages=5, page_size=100) @@ -461,17 +461,17 @@ All access layer methods have async versions (prefixed with ``a``): async def main(): bdl = BDL() - + # Async methods return DataFrames levels_df = await bdl.levels.alist_levels() variables_df = await bdl.variables.alist_variables() - + # Can run multiple requests concurrently levels_task = bdl.levels.alist_levels() variables_task = bdl.variables.alist_variables() - + levels_df, variables_df = await asyncio.gather(levels_task, variables_task) - + return levels_df, variables_df asyncio.run(main()) @@ -517,10 +517,10 @@ Filtering and Analysis # Get all variables variables = bdl.variables.list_variables() - + # Filter using pandas economic_vars = variables[variables['name'].str.contains('economic', case=False)] - + # Get data for multiple variables for var_id in economic_vars['id'].head(5): data = bdl.data.get_data_by_variable(var_id, years=[2021]) @@ -533,10 +533,10 @@ Getting Data # Get data df = bdl.data.get_data_by_variable("3643", years=[2021]) - + # DataFrame includes IDs and values print(df[['unit_name', 'attr_name', 'val']].head()) - + # Group by attribute name by_attr = df.groupby('attr_name')['val'].mean() print(by_attr) @@ -550,15 +550,15 @@ The data endpoints automatically normalize nested structures: # API returns nested structure, but access layer flattens it df = bdl.data.get_data_by_variable("3643", years=[2021]) - + # Each row represents one data point # Columns: unit_id, unit_name, year, val, attr_id, attr_name print(df.head()) - + # Easy to analyze avg_by_unit = df.groupby('unit_name')['val'].mean() print(avg_by_unit) - + # Get data for multiple years multi_year_df = bdl.data.get_data_by_variable("3643", years=[2020, 2021, 2022]) # Analyze trends over time @@ -674,4 +674,3 @@ Years - :doc:`api_clients` for low-level API access - :doc:`examples` for real-world examples - :doc:`config` for configuration options - diff --git a/docs/api_clients.rst b/docs/api_clients.rst index a3d8d75..b473ad2 100644 --- a/docs/api_clients.rst +++ b/docs/api_clients.rst @@ -103,16 +103,16 @@ The format and language parameters automatically set the appropriate HTTP header .. code-block:: python from pybdl import BDL - + bdl = BDL() - + # Request data in XML format data = bdl.api.data.get_data_by_variable( variable_id="3643", years=[2021], format="xml" ) - + # Request data in Polish data = bdl.api.data.get_data_by_variable( variable_id="3643", @@ -218,4 +218,4 @@ Years :undoc-members: :show-inheritance: :inherited-members: - :noindex: \ No newline at end of file + :noindex: diff --git a/docs/appendix.rst b/docs/appendix.rst index 4c93dab..5a51a5f 100644 --- a/docs/appendix.rst +++ b/docs/appendix.rst @@ -217,7 +217,7 @@ DataFrame Conversion The access layer converts API responses to pandas DataFrames through several steps: 1. **Column Name Normalization**: camelCase → snake_case using regex patterns -2. **Data Type Inference**: +2. **Data Type Inference**: - Attempts numeric conversion (int/float) - Detects boolean values - Preserves strings/objects @@ -238,7 +238,7 @@ Example transformation: # API response: [{"id": "1", "name": "Warsaw", "values": [{"year": 2021, "val": 1000}]}] - + # Access layer output: # DataFrame with columns: unit_id, unit_name, year, val @@ -285,4 +285,3 @@ Error Handling - :doc:`config` for user-facing configuration documentation - :doc:`api_clients` for API client usage - :doc:`access_layer` for access layer documentation - diff --git a/docs/config.rst b/docs/config.rst index 48dd241..d70f408 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -19,10 +19,10 @@ Basic Setup .. code-block:: python from pybdl import BDL, BDLConfig - + # Minimal configuration (reads API key from environment) bdl = BDL() - + # Or provide API key directly config = BDLConfig(api_key="your-api-key") bdl = BDL(config) @@ -35,14 +35,14 @@ The API supports anonymous access without an API key. When ``api_key`` is explic .. code-block:: python from pybdl import BDL, BDLConfig - + # Anonymous access (explicitly None - overrides environment variables) config = BDLConfig(api_key=None) bdl = BDL(config) - + # Or simply pass None bdl = BDL(config=None) # Creates default config with api_key=None - + # Or use dict bdl = BDL(config={"api_key": None}) @@ -208,10 +208,10 @@ At a high level, caching works like this: # Enable file-backed caching with 10-minute expiry config = BDLConfig(api_key="...", cache_backend="file", cache_expire_after=600) bdl = BDL(config) - + # First call hits the API data1 = bdl.data.get_data_by_variable("3643", years=[2021]) - + # Second call uses cache (if within expiry time) data2 = bdl.data.get_data_by_variable("3643", years=[2021]) diff --git a/docs/examples.ipynb b/docs/examples.ipynb index 6233cc2..662b29e 100644 --- a/docs/examples.ipynb +++ b/docs/examples.ipynb @@ -1,802 +1,801 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Examples\n", - "\n", - "This section provides real-world examples demonstrating how to use pyBDL for common data analysis tasks.\n" - ] - }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Examples\n", + "\n", + "This section provides real-world examples demonstrating how to use pyBDL for common data analysis tasks.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Data Retrieval\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Started\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Data Retrieval\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching levels: 1 pages [00:00, 11.46 pages/s, items=8]\n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Getting Started\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Administrative levels:\n", + " id name\n", + "0 0 Poziom Polski\n", + "1 1 Poziom Makroregionów\n", + "2 2 Poziom Województw\n", + "3 3 Poziom Regionów\n", + "4 4 Poziom Podregionów\n", + "5 5 Poziom Powiatów\n", + "6 6 Poziom Gmin\n", + "7 7 Poziom miejscowości statystycznej\n" + ] }, { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching levels: 1 pages [00:00, 11.46 pages/s, items=8]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Administrative levels:\n", - " id name\n", - "0 0 Poziom Polski\n", - "1 1 Poziom Makroregionów\n", - "2 2 Poziom Województw\n", - "3 3 Poziom Regionów\n", - "4 4 Poziom Podregionów\n", - "5 5 Poziom Powiatów\n", - "6 6 Poziom Gmin\n", - "7 7 Poziom miejscowości statystycznej\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching years: 1 pages [00:00, 1.47 pages/s, items=32]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Available years: 1995 - 2026\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "from pybdl import BDL, BDLConfig\n", - "\n", - "# Initialize client\n", - "bdl = BDL()\n", - "\n", - "# List available administrative levels\n", - "levels = bdl.levels.list_levels()\n", - "print(\"Administrative levels:\")\n", - "print(levels[['id', 'name']])\n", - "\n", - "# List available years\n", - "years = bdl.years.list_years()\n", - "print(f\"\\nAvailable years: {years['id'].min()} - {years['id'].max()}\")" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching years: 1 pages [00:00, 1.47 pages/s, items=32]" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Finding Variables\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Available years: 1995 - 2026\n" + ] }, { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching search: 18 pages [01:44, 5.82s/ pages, items=1760]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found 1760 population-related variables\n", - " id n1\n", - "0 9179 concerning self-taxation of the population\n", - "1 1365239 total net migration per 1000 population\n", - "2 498816 net migration in internal movement per 1000 po...\n", - "3 745534 net migration abroad per 1000 population\n", - "4 453193 net migration per 1000 population\n", - "\n", - "Variable details:\n", - " id n1 n2\n", - "0 3643 total 30-39\n" - ] - } - ], - "source": [ - "# Search for population-related variables\n", - "population_vars = bdl.variables.search_variables(name=\"population\")\n", - "print(f\"Found {len(population_vars)} population-related variables\")\n", - "print(population_vars[['id', 'n1']].head())\n", - "\n", - "# Get details for a specific variable\n", - "var_details = bdl.variables.get_variable(\"3643\")\n", - "print(f\"\\nVariable details:\")\n", - "print(var_details[['id', 'n1', 'n2']])\n" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "from pybdl import BDL\n", + "\n", + "# Initialize client\n", + "bdl = BDL()\n", + "\n", + "# List available administrative levels\n", + "levels = bdl.levels.list_levels()\n", + "print(\"Administrative levels:\")\n", + "print(levels[['id', 'name']])\n", + "\n", + "# List available years\n", + "years = bdl.years.list_years()\n", + "print(f\"\\nAvailable years: {years['id'].min()} - {years['id'].max()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Finding Variables\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Retrieving Data\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching search: 18 pages [01:44, 5.82s/ pages, items=1760]\n" + ] }, { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching 3643: 1 pages [00:00, 2.57 pages/s, items=16]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Retrieved 16 data points\n", - " unit_name year val\n", - "0 MAŁOPOLSKIE 2021 2\n", - "1 ŚLĄSKIE 2021 6\n", - "2 LUBUSKIE 2021 0\n", - "3 WIELKOPOLSKIE 2021 2\n", - "4 ZACHODNIOPOMORSKIE 2021 5\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Get data for a variable at voivodeship level (level 2)\n", - "data = bdl.data.get_data_by_variable(\n", - " variable_id=\"3643\",\n", - " years=[2021],\n", - " unit_level=2 # Voivodeship level\n", - ")\n", - "\n", - "print(f\"Retrieved {len(data)} data points\")\n", - "print(data[['unit_name', 'year', 'val']].head())\n" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 1760 population-related variables\n", + " id n1\n", + "0 9179 concerning self-taxation of the population\n", + "1 1365239 total net migration per 1000 population\n", + "2 498816 net migration in internal movement per 1000 po...\n", + "3 745534 net migration abroad per 1000 population\n", + "4 453193 net migration per 1000 population\n", + "\n", + "Variable details:\n", + " id n1 n2\n", + "0 3643 total 30-39\n" + ] + } + ], + "source": [ + "# Search for population-related variables\n", + "population_vars = bdl.variables.search_variables(name=\"population\")\n", + "print(f\"Found {len(population_vars)} population-related variables\")\n", + "print(population_vars[['id', 'n1']].head())\n", + "\n", + "# Get details for a specific variable\n", + "var_details = bdl.variables.get_variable(\"3643\")\n", + "print(\"\\nVariable details:\")\n", + "print(var_details[['id', 'n1', 'n2']])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Retrieving Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching aggregates: 1 pages [00:00, 13.47 pages/s, items=8]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " id name level \\\n", - "0 1 TOTAL 7 \n", - "1 2 URBAN GMINAS 5 \n", - "2 3 URBAN-RURAL GMINAS 5 \n", - "3 4 RURAL GMINAS 5 \n", - "4 7 URBAN AREAS 5 \n", - "5 8 RURAL AREAS 5 \n", - "6 91 NP- Górnośląsko-Zagłębiowska Metropolia 3 \n", - "7 92 NP- Metropolia Krakowska 3 \n", - "\n", - " description \n", - "0 Aggregates for items collected on the level: <... \n", - "1 It is a sum of data for urban gminas (unit typ... \n", - "2 It is a sum of data for urban-rural gminas (un... \n", - "3 It is a sum of these data for rural gminas (un... \n", - "4 It is a sum of data for urban areas according... \n", - "5 It is a sum of data for rural areas, i.e. rura... \n", - "6 NaN \n", - "7 NaN \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "data_aggr = bdl.aggregates.list_aggregates()\n", - "print(data_aggr)" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching 3643: 1 pages [00:00, 2.57 pages/s, items=16]" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Population Analysis by Region\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Retrieved 16 data points\n", + " unit_name year val\n", + "0 MAŁOPOLSKIE 2021 2\n", + "1 ŚLĄSKIE 2021 6\n", + "2 LUBUSKIE 2021 0\n", + "3 WIELKOPOLSKIE 2021 2\n", + "4 ZACHODNIOPOMORSKIE 2021 5\n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Finding Population Variables\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Get data for a variable at voivodeship level (level 2)\n", + "data = bdl.data.get_data_by_variable(\n", + " variable_id=\"3643\",\n", + " years=[2021],\n", + " unit_level=2 # Voivodeship level\n", + ")\n", + "\n", + "print(f\"Retrieved {len(data)} data points\")\n", + "print(data[['unit_name', 'year', 'val']].head())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching aggregates: 1 pages [00:00, 13.47 pages/s, items=8]" + ] }, { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching search: 18 pages [00:00, 284.41 pages/s, items=1760]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found 321 total population variables\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Search for population variables\n", - "pop_vars = bdl.variables.search_variables(name=\"population\")\n", - "\n", - "# Filter for total population (usually contains \"total\" or \"ogółem\")\n", - "total_pop = pop_vars[\n", - " pop_vars['n2'].str.contains('total', case=False, na=False)\n", - "]\n", - "print(f\"Found {len(total_pop)} total population variables\")\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + " id name level \\\n", + "0 1 TOTAL 7 \n", + "1 2 URBAN GMINAS 5 \n", + "2 3 URBAN-RURAL GMINAS 5 \n", + "3 4 RURAL GMINAS 5 \n", + "4 7 URBAN AREAS 5 \n", + "5 8 RURAL AREAS 5 \n", + "6 91 NP- Górnośląsko-Zagłębiowska Metropolia 3 \n", + "7 92 NP- Metropolia Krakowska 3 \n", + "\n", + " description \n", + "0 Aggregates for items collected on the level: <... \n", + "1 It is a sum of data for urban gminas (unit typ... \n", + "2 It is a sum of data for urban-rural gminas (un... \n", + "3 It is a sum of these data for rural gminas (un... \n", + "4 It is a sum of data for urban areas according... \n", + "5 It is a sum of data for rural areas, i.e. rura... \n", + "6 NaN \n", + "7 NaN \n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Getting Regional Population Data\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "data_aggr = bdl.aggregates.list_aggregates()\n", + "print(data_aggr)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Population Analysis by Region\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Finding Population Variables\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching search: 18 pages [00:00, 284.41 pages/s, items=1760]" + ] }, { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching 3643: 1 pages [00:00, 235.52 pages/s, items=16]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Top 5 voivodeships by population:\n", - " unit_name val\n", - "15 MAZOWIECKIE 12\n", - "1 ŚLĄSKIE 6\n", - "4 ZACHODNIOPOMORSKIE 5\n", - "8 POMORSKIE 4\n", - "10 ŁÓDZKIE 4\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Get population data for all voivodeships in 2021\n", - "pop_data = bdl.data.get_data_by_variable(\n", - " variable_id=\"3643\", # Example: total population variable\n", - " years=[2021],\n", - " unit_level=2 # Voivodeship level\n", - ")\n", - "\n", - "# Sort by population\n", - "pop_sorted = pop_data.sort_values('val', ascending=False)\n", - "print(\"Top 5 voivodeships by population:\")\n", - "print(pop_sorted[['unit_name', 'val']].head())\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 321 total population variables\n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Economic Indicator Comparison\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Search for population variables\n", + "pop_vars = bdl.variables.search_variables(name=\"population\")\n", + "\n", + "# Filter for total population (usually contains \"total\" or \"ogółem\")\n", + "total_pop = pop_vars[\n", + " pop_vars['n2'].str.contains('total', case=False, na=False)\n", + "]\n", + "print(f\"Found {len(total_pop)} total population variables\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Regional Population Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching 3643: 1 pages [00:00, 235.52 pages/s, items=16]" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Finding Economic Variables\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Top 5 voivodeships by population:\n", + " unit_name val\n", + "15 MAZOWIECKIE 12\n", + "1 ŚLĄSKIE 6\n", + "4 ZACHODNIOPOMORSKIE 5\n", + "8 POMORSKIE 4\n", + "10 ŁÓDZKIE 4\n" + ] }, { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching search: 5 pages [00:28, 5.76s/ pages, items=458]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found 458 unemployment variables\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching search: 1 pages [00:05, 5.25s/ pages, items=11]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found 11 GDP-related variables\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Search for unemployment variables\n", - "unemployment_vars = bdl.variables.search_variables(name=\"unemployment\")\n", - "print(f\"Found {len(unemployment_vars)} unemployment variables\")\n", - "\n", - "# Search for GDP-related variables\n", - "gdp_vars = bdl.variables.search_variables(name=\"GDP\")\n", - "print(f\"Found {len(gdp_vars)} GDP-related variables\")\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Get population data for all voivodeships in 2021\n", + "pop_data = bdl.data.get_data_by_variable(\n", + " variable_id=\"3643\", # Example: total population variable\n", + " years=[2021],\n", + " unit_level=2 # Voivodeship level\n", + ")\n", + "\n", + "# Sort by population\n", + "pop_sorted = pop_data.sort_values('val', ascending=False)\n", + "print(\"Top 5 voivodeships by population:\")\n", + "print(pop_sorted[['unit_name', 'val']].head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Economic Indicator Comparison\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Finding Economic Variables\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching search: 5 pages [00:28, 5.76s/ pages, items=458]\n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Comparing Voivodeships\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 458 unemployment variables\n" + ] }, { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching 1234: 1 pages [00:00, 5.28 pages/s, items=16]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unemployment by voivodeship (2021):\n", - " unit_name val\n", - "15 MAZOWIECKIE 766241\n", - "1 ŚLĄSKIE 594875\n", - "0 MAŁOPOLSKIE 544790\n", - "3 WIELKOPOLSKIE 411015\n", - "5 DOLNOŚLĄSKIE 334400\n", - "10 ŁÓDZKIE 315051\n", - "8 POMORSKIE 297680\n", - "12 LUBELSKIE 286248\n", - "13 PODKARPACKIE 268623\n", - "7 KUJAWSKO-POMORSKIE 199304\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Get unemployment data for all voivodeships\n", - "unemployment_data = bdl.data.get_data_by_variable(\n", - " variable_id=\"1234\", # Example unemployment variable ID\n", - " years=[2021],\n", - " unit_level=2\n", - ")\n", - "\n", - "# Sort and display\n", - "sorted_unemployment = unemployment_data.sort_values('val', ascending=False)\n", - "print(\"Unemployment by voivodeship (2021):\")\n", - "print(sorted_unemployment[['unit_name', 'val']].head(10))\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching search: 1 pages [00:05, 5.25s/ pages, items=11]" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Time Series Analysis\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 11 GDP-related variables\n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Preparing Data for Visualization\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Search for unemployment variables\n", + "unemployment_vars = bdl.variables.search_variables(name=\"unemployment\")\n", + "print(f\"Found {len(unemployment_vars)} unemployment variables\")\n", + "\n", + "# Search for GDP-related variables\n", + "gdp_vars = bdl.variables.search_variables(name=\"GDP\")\n", + "print(f\"Found {len(gdp_vars)} GDP-related variables\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Comparing Voivodeships\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching 1234: 1 pages [00:00, 5.28 pages/s, items=16]" + ] }, { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching 3643: 1 pages [00:00, 8.17 pages/s, items=3]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " val\n", - "year \n", - "2015 2\n", - "2015 3\n", - "2015 3\n", - "2016 1\n", - "2016 2\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Get time series data for a specific variable and region\n", - "time_series = bdl.data.get_data_by_variable(\n", - " variable_id=\"3643\",\n", - " unit_level=2,\n", - " unit_parent_id=\"020000000000\" # Example: Mazovian Voivodeship\n", - ")\n", - "\n", - "# Filter to recent years\n", - "recent_years = time_series[time_series['year'] >= 2015]\n", - "\n", - "# Sort by year\n", - "recent_years = recent_years.sort_values('year')\n", - "\n", - "# Prepare for plotting\n", - "plot_data = recent_years[['year', 'val']].set_index('year')\n", - "print(plot_data.head())\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Unemployment by voivodeship (2021):\n", + " unit_name val\n", + "15 MAZOWIECKIE 766241\n", + "1 ŚLĄSKIE 594875\n", + "0 MAŁOPOLSKIE 544790\n", + "3 WIELKOPOLSKIE 411015\n", + "5 DOLNOŚLĄSKIE 334400\n", + "10 ŁÓDZKIE 315051\n", + "8 POMORSKIE 297680\n", + "12 LUBELSKIE 286248\n", + "13 PODKARPACKIE 268623\n", + "7 KUJAWSKO-POMORSKIE 199304\n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Error Handling\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Get unemployment data for all voivodeships\n", + "unemployment_data = bdl.data.get_data_by_variable(\n", + " variable_id=\"1234\", # Example unemployment variable ID\n", + " years=[2021],\n", + " unit_level=2\n", + ")\n", + "\n", + "# Sort and display\n", + "sorted_unemployment = unemployment_data.sort_values('val', ascending=False)\n", + "print(\"Unemployment by voivodeship (2021):\")\n", + "print(sorted_unemployment[['unit_name', 'val']].head(10))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Time Series Analysis\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Preparing Data for Visualization\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching 3643: 1 pages [00:00, 8.17 pages/s, items=3]" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Handling Missing Data\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + " val\n", + "year \n", + "2015 2\n", + "2015 3\n", + "2015 3\n", + "2016 1\n", + "2016 2\n" + ] }, { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fetching 3643: 1 pages [00:00, 258.60 pages/s, items=16]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found 16 records\n", - " year val unit_id unit_name attr_id\n", - "0 2021 2 11200000000 MAŁOPOLSKIE 1\n", - "1 2021 6 12400000000 ŚLĄSKIE 1\n", - "2 2021 0 20800000000 LUBUSKIE 0\n", - "3 2021 2 23000000000 WIELKOPOLSKIE 1\n", - "4 2021 5 23200000000 ZACHODNIOPOMORSKIE 1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Check if data exists before processing\n", - "data = bdl.data.get_data_by_variable(\"3643\", years=[2021], unit_level=2)\n", - "\n", - "if data.empty:\n", - " print(\"No data available for this variable/year/level combination\")\n", - "else:\n", - " # Process data\n", - " print(f\"Found {len(data)} records\")\n", - " print(data.head())\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "\n", + "# Get time series data for a specific variable and region\n", + "time_series = bdl.data.get_data_by_variable(\n", + " variable_id=\"3643\",\n", + " unit_level=2,\n", + " unit_parent_id=\"020000000000\" # Example: Mazovian Voivodeship\n", + ")\n", + "\n", + "# Filter to recent years\n", + "recent_years = time_series[time_series['year'] >= 2015]\n", + "\n", + "# Sort by year\n", + "recent_years = recent_years.sort_values('year')\n", + "\n", + "# Prepare for plotting\n", + "plot_data = recent_years[['year', 'val']].set_index('year')\n", + "print(plot_data.head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Error Handling\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Handling Missing Data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fetching 3643: 1 pages [00:00, 258.60 pages/s, items=16]" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Enrichment: Adding Human-Readable Labels\n", - "\n", - "The access layer can automatically join reference data (levels, attributes, subjects, etc.)\n", - "onto the result DataFrame via the `enrich` parameter. Enrichment fetches each lookup table\n", - "once per session and caches it in memory.\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 16 records\n", + " year val unit_id unit_name attr_id\n", + "0 2021 2 11200000000 MAŁOPOLSKIE 1\n", + "1 2021 6 12400000000 ŚLĄSKIE 1\n", + "2 2021 0 20800000000 LUBUSKIE 0\n", + "3 2021 2 23000000000 WIELKOPOLSKIE 1\n", + "4 2021 5 23200000000 ZACHODNIOPOMORSKIE 1\n" + ] }, { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Fetching levels: 1 pages [00:00, 202.55 pages/s, items=8]\n", - "Fetching measures: 1 pages [00:00, 200.37 pages/s, items=75]\n", - "Fetching subjects: 1 pages [00:00, 243.37 pages/s, items=33]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Variables with enrichment:\n", - " id_x n1 level \\\n", - "0 9179 concerning self-taxation of the population 5 \n", - "1 1365239 total net migration per 1000 population 6 \n", - "2 498816 net migration in internal movement per 1000 po... 6 \n", - "3 745534 net migration abroad per 1000 population 6 \n", - "4 453193 net migration per 1000 population 6 \n", - "\n", - " level_name measure_unit_id measure_unit_description \n", - "0 Poziom Powiatów 8 number of pieces \n", - "1 Poziom Gmin 26 number of persons \n", - "2 Poziom Gmin 26 number of persons \n", - "3 Poziom Gmin 26 number of persons \n", - "4 Poziom Gmin 26 number of persons \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "from pybdl import BDL\n", - "\n", - "bdl = BDL()\n", - "\n", - "# Enrich variables with level names, measure descriptions, and subject names.\n", - "# Without enrichment, the DataFrame only contains raw IDs (level, measure_unit_id, subject_id).\n", - "variables = bdl.variables.search_variables(name=\"population\", max_pages=1, enrich=[\"levels\", \"measures\", \"subjects\"])\n", - "print(\"Variables with enrichment:\")\n", - "print(variables[[\"id_x\", \"n1\", \"level\", \"level_name\", \"measure_unit_id\", \"measure_unit_description\"]].head())\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Check if data exists before processing\n", + "data = bdl.data.get_data_by_variable(\"3643\", years=[2021], unit_level=2)\n", + "\n", + "if data.empty:\n", + " print(\"No data available for this variable/year/level combination\")\n", + "else:\n", + " # Process data\n", + " print(f\"Found {len(data)} records\")\n", + " print(data.head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Enrichment: Adding Human-Readable Labels\n", + "\n", + "The access layer can automatically join reference data (levels, attributes, subjects, etc.)\n", + "onto the result DataFrame via the `enrich` parameter. Enrichment fetches each lookup table\n", + "once per session and caches it in memory.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Fetching levels: 1 pages [00:00, 202.55 pages/s, items=8]\n", + "Fetching measures: 1 pages [00:00, 200.37 pages/s, items=75]\n", + "Fetching subjects: 1 pages [00:00, 243.37 pages/s, items=33]" + ] }, { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Fetching 3643: 1 pages [00:00, 215.93 pages/s, items=16]\n", - "Fetching units: 10 pages [11:54, 71.46s/ pages, items=1000] \n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Enrich data with attribute labels and unit details.\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;66;03m# 'attributes' adds attr_name / attr_symbol columns; 'units' adds unit_name_enriched, unit_level, etc.\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m data = \u001b[43mbdl\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_data_by_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[43m \u001b[49m\u001b[43mvariable_id\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m3643\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 5\u001b[39m \u001b[43m \u001b[49m\u001b[43myears\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2021\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 6\u001b[39m \u001b[43m \u001b[49m\u001b[43munit_level\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 7\u001b[39m \u001b[43m \u001b[49m\u001b[43menrich\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mattributes\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43munits\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 9\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mData with enriched attribute and unit columns:\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 10\u001b[39m \u001b[38;5;28mprint\u001b[39m(data[[\u001b[33m\"\u001b[39m\u001b[33munit_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33munit_name_enriched\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33myear\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mval\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mattr_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mattr_name\u001b[39m\u001b[33m\"\u001b[39m]].head())\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/access/enrichment.py:397\u001b[39m, in \u001b[36mwith_enrichment..decorator..wrapper\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 395\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m spec \u001b[38;5;129;01min\u001b[39;00m specs:\n\u001b[32m 396\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m flags.get(spec.flag):\n\u001b[32m--> \u001b[39m\u001b[32m397\u001b[39m lookup_df = \u001b[43m_get_lookup\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspec\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 398\u001b[39m df = _merge_enrichment(df, lookup_df, spec)\n\u001b[32m 399\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _recombine_result(df, metadata)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/access/enrichment.py:295\u001b[39m, in \u001b[36m_get_lookup\u001b[39m\u001b[34m(access, spec)\u001b[39m\n\u001b[32m 293\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m spec.cache_key \u001b[38;5;129;01min\u001b[39;00m cache:\n\u001b[32m 294\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m cache[spec.cache_key]\n\u001b[32m--> \u001b[39m\u001b[32m295\u001b[39m lookup_df = \u001b[43mspec\u001b[49m\u001b[43m.\u001b[49m\u001b[43msync_loader\u001b[49m\u001b[43m(\u001b[49m\u001b[43maccess\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 296\u001b[39m cache[spec.cache_key] = lookup_df\n\u001b[32m 297\u001b[39m access._enrichment_cache = cache\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/access/enrichment.py:122\u001b[39m, in \u001b[36m_fetch_units_sync\u001b[39m\u001b[34m(access)\u001b[39m\n\u001b[32m 119\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 120\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpybdl\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01munits\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m UnitsAPI\n\u001b[32m--> \u001b[39m\u001b[32m122\u001b[39m data = \u001b[43mUnitsAPI\u001b[49m\u001b[43m(\u001b[49m\u001b[43maccess\u001b[49m\u001b[43m.\u001b[49m\u001b[43mapi_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlist_units\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43maccess\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_get_default_page_size\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m 123\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _normalize_lookup_dataframe(access, data)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/units.py:108\u001b[39m, in \u001b[36mUnitsAPI.list_units\u001b[39m\u001b[34m(self, parent_id, level, page, page_size, max_pages, sort, lang, format, if_none_match, if_modified_since, extra_query)\u001b[39m\n\u001b[32m 94\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mlist_units\u001b[39m(\n\u001b[32m 95\u001b[39m \u001b[38;5;28mself\u001b[39m,\n\u001b[32m 96\u001b[39m parent_id: \u001b[38;5;28mstr\u001b[39m | \u001b[38;5;28;01mNone\u001b[39;00m = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m (...)\u001b[39m\u001b[32m 106\u001b[39m extra_query: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any] | \u001b[38;5;28;01mNone\u001b[39;00m = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m 107\u001b[39m ) -> \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any]]:\n\u001b[32m--> \u001b[39m\u001b[32m108\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_fetch_collection_endpoint\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 109\u001b[39m \u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43munits\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 110\u001b[39m \u001b[43m \u001b[49m\u001b[43mextra_params\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_list_units_params\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparent_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msort\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_query\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 111\u001b[39m \u001b[43m \u001b[49m\u001b[43mlang\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlang\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 112\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 113\u001b[39m \u001b[43m \u001b[49m\u001b[43mif_none_match\u001b[49m\u001b[43m=\u001b[49m\u001b[43mif_none_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 114\u001b[39m \u001b[43m \u001b[49m\u001b[43mif_modified_since\u001b[49m\u001b[43m=\u001b[49m\u001b[43mif_modified_since\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 115\u001b[39m \u001b[43m \u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 116\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 117\u001b[39m \u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mresults\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 118\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:738\u001b[39m, in \u001b[36mBaseAPIClient._fetch_collection_endpoint\u001b[39m\u001b[34m(self, endpoint, extra_params, lang, format, if_none_match, if_modified_since, page_size, max_pages, results_key)\u001b[39m\n\u001b[32m 730\u001b[39m params_with_page_size[\u001b[33m\"\u001b[39m\u001b[33mpage-size\u001b[39m\u001b[33m\"\u001b[39m] = page_size\n\u001b[32m 731\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m.fetch_single_result(\n\u001b[32m 732\u001b[39m endpoint,\n\u001b[32m 733\u001b[39m results_key=results_key,\n\u001b[32m 734\u001b[39m params=params_with_page_size,\n\u001b[32m 735\u001b[39m headers=headers,\n\u001b[32m 736\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m738\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mfetch_all_results\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 739\u001b[39m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 740\u001b[39m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m=\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 741\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 742\u001b[39m \u001b[43m \u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 743\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 744\u001b[39m \u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 745\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:536\u001b[39m, in \u001b[36mBaseAPIClient.fetch_all_results\u001b[39m\u001b[34m(self, endpoint, method, params, headers, results_key, page_size, max_pages, return_metadata, show_progress)\u001b[39m\n\u001b[32m 534\u001b[39m first_page = \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m 535\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m536\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpage\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_paginated_request_sync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 537\u001b[39m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 538\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 539\u001b[39m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m=\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 540\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 541\u001b[39m \u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 542\u001b[39m \u001b[43m \u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 543\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 544\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 545\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpage\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 546\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mraise\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mBDLResponseError\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mResponse does not contain key \u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mresults_key\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m'\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpayload\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:451\u001b[39m, in \u001b[36mBaseAPIClient._paginated_request_sync\u001b[39m\u001b[34m(self, endpoint, method, params, headers, results_key, page_size, max_pages, return_all)\u001b[39m\n\u001b[32m 449\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m next_url:\n\u001b[32m 450\u001b[39m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m451\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_request_sync_url\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnext_url\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 453\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m results_key \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m resp:\n\u001b[32m 454\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m BDLResponseError(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mResponse does not contain key \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresults_key\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m\"\u001b[39m, payload=resp)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:363\u001b[39m, in \u001b[36mBaseAPIClient._request_sync_url\u001b[39m\u001b[34m(self, url, method, params, headers)\u001b[39m\n\u001b[32m 361\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m retries_429 < \u001b[38;5;28mself\u001b[39m.config.http_429_max_retries:\n\u001b[32m 362\u001b[39m retries_429 += \u001b[32m1\u001b[39m\n\u001b[32m--> \u001b[39m\u001b[32m363\u001b[39m \u001b[43mtime\u001b[49m\u001b[43m.\u001b[49m\u001b[43msleep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_retry_delay_after_429\u001b[49m\u001b[43m(\u001b[49m\u001b[43mretries_429\u001b[49m\u001b[43m \u001b[49m\u001b[43m-\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 364\u001b[39m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[32m 365\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._process_response(response)\n", - "\u001b[31mKeyboardInterrupt\u001b[39m: " - ] - } - ], - "source": [ - "# Enrich data with attribute labels and unit details.\n", - "# 'attributes' adds attr_name / attr_symbol columns; 'units' adds unit_name_enriched, unit_level, etc.\n", - "data = bdl.data.get_data_by_variable(\n", - " variable_id=\"3643\",\n", - " years=[2021],\n", - " unit_level=2,\n", - " enrich=[\"attributes\", \"units\"],\n", - ")\n", - "print(\"Data with enriched attribute and unit columns:\")\n", - "print(data[[\"unit_id\", \"unit_name_enriched\", \"year\", \"val\", \"attr_id\", \"attr_name\"]].head())\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Variables with enrichment:\n", + " id_x n1 level \\\n", + "0 9179 concerning self-taxation of the population 5 \n", + "1 1365239 total net migration per 1000 population 6 \n", + "2 498816 net migration in internal movement per 1000 po... 6 \n", + "3 745534 net migration abroad per 1000 population 6 \n", + "4 453193 net migration per 1000 population 6 \n", + "\n", + " level_name measure_unit_id measure_unit_description \n", + "0 Poziom Powiatów 8 number of pieces \n", + "1 Poziom Gmin 26 number of persons \n", + "2 Poziom Gmin 26 number of persons \n", + "3 Poziom Gmin 26 number of persons \n", + "4 Poziom Gmin 26 number of persons \n" + ] }, { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Fetching 3643: 1 pages [00:00, 161.21 pages/s, items=16]\n", - "Fetching attributes: 1 pages [00:00, 15.45 pages/s, items=18]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total pages: None, Total records: 16\n", - " unit_id year val attr_name\n", - "0 11200000000 2021 2 value\n", - "1 12400000000 2021 6 value\n", - "2 20800000000 2021 0 0\n", - "3 23000000000 2021 2 value\n", - "4 23200000000 2021 5 value\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# Combine enrichment with metadata retrieval.\n", - "# return_metadata=True returns a (DataFrame, metadata_dict) tuple.\n", - "df, meta = bdl.data.get_data_by_variable(\n", - " variable_id=\"3643\",\n", - " years=[2021],\n", - " unit_level=2,\n", - " enrich=[\"attributes\"],\n", - " return_metadata=True,\n", - ")\n", - "print(f\"Total pages: {meta.get('totalPages')}, Total records: {meta.get('totalRecords')}\")\n", - "print(df[[\"unit_id\", \"year\", \"val\", \"attr_name\"]].head())\n" - ] + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "from pybdl import BDL\n", + "\n", + "bdl = BDL()\n", + "\n", + "# Enrich variables with level names, measure descriptions, and subject names.\n", + "# Without enrichment, the DataFrame only contains raw IDs (level, measure_unit_id, subject_id).\n", + "variables = bdl.variables.search_variables(name=\"population\", max_pages=1, enrich=[\"levels\", \"measures\", \"subjects\"])\n", + "print(\"Variables with enrichment:\")\n", + "print(variables[[\"id_x\", \"n1\", \"level\", \"level_name\", \"measure_unit_id\", \"measure_unit_description\"]].head())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Fetching 3643: 1 pages [00:00, 215.93 pages/s, items=16]\n", + "Fetching units: 10 pages [11:54, 71.46s/ pages, items=1000] \n" + ] }, { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Best Practices\n", - "\n", - "1. **Use the access layer**: Prefer ``bdl.data`` over ``bdl.api.data`` for DataFrame output\n", - "2. **Enable caching**: Use ``use_cache=True`` for repeated queries\n", - "3. **Handle pagination**: Use ``max_pages=None`` to get complete datasets\n", - "4. **Use enrichment**: Let the library automatically add human-readable names\n", - "5. **Async for bulk operations**: Use async methods when fetching multiple datasets\n", - "6. **Error handling**: Always handle ``RateLimitError`` and check for empty results\n", - "\n", - "```{seealso}\n", - "- {doc}`access_layer` for access layer documentation\n", - "- {doc}`rate_limiting` for rate limiting details\n", - "- {doc}`config` for configuration options\n", - "```\n" - ] + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[2]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;66;03m# Enrich data with attribute labels and unit details.\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;66;03m# 'attributes' adds attr_name / attr_symbol columns; 'units' adds unit_name_enriched, unit_level, etc.\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m data = \u001b[43mbdl\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_data_by_variable\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[43m \u001b[49m\u001b[43mvariable_id\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m3643\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 5\u001b[39m \u001b[43m \u001b[49m\u001b[43myears\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[32;43m2021\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 6\u001b[39m \u001b[43m \u001b[49m\u001b[43munit_level\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m2\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 7\u001b[39m \u001b[43m \u001b[49m\u001b[43menrich\u001b[49m\u001b[43m=\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mattributes\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43munits\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[43m)\u001b[49m\n\u001b[32m 9\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mData with enriched attribute and unit columns:\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 10\u001b[39m \u001b[38;5;28mprint\u001b[39m(data[[\u001b[33m\"\u001b[39m\u001b[33munit_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33munit_name_enriched\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33myear\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mval\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mattr_id\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mattr_name\u001b[39m\u001b[33m\"\u001b[39m]].head())\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/access/enrichment.py:397\u001b[39m, in \u001b[36mwith_enrichment..decorator..wrapper\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 395\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m spec \u001b[38;5;129;01min\u001b[39;00m specs:\n\u001b[32m 396\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m flags.get(spec.flag):\n\u001b[32m--> \u001b[39m\u001b[32m397\u001b[39m lookup_df = \u001b[43m_get_lookup\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspec\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 398\u001b[39m df = _merge_enrichment(df, lookup_df, spec)\n\u001b[32m 399\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _recombine_result(df, metadata)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/access/enrichment.py:295\u001b[39m, in \u001b[36m_get_lookup\u001b[39m\u001b[34m(access, spec)\u001b[39m\n\u001b[32m 293\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m spec.cache_key \u001b[38;5;129;01min\u001b[39;00m cache:\n\u001b[32m 294\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m cache[spec.cache_key]\n\u001b[32m--> \u001b[39m\u001b[32m295\u001b[39m lookup_df = \u001b[43mspec\u001b[49m\u001b[43m.\u001b[49m\u001b[43msync_loader\u001b[49m\u001b[43m(\u001b[49m\u001b[43maccess\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 296\u001b[39m cache[spec.cache_key] = lookup_df\n\u001b[32m 297\u001b[39m access._enrichment_cache = cache\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/access/enrichment.py:122\u001b[39m, in \u001b[36m_fetch_units_sync\u001b[39m\u001b[34m(access)\u001b[39m\n\u001b[32m 119\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 120\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpybdl\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mapi\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01munits\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m UnitsAPI\n\u001b[32m--> \u001b[39m\u001b[32m122\u001b[39m data = \u001b[43mUnitsAPI\u001b[49m\u001b[43m(\u001b[49m\u001b[43maccess\u001b[49m\u001b[43m.\u001b[49m\u001b[43mapi_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconfig\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlist_units\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43maccess\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_get_default_page_size\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m 123\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _normalize_lookup_dataframe(access, data)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/units.py:108\u001b[39m, in \u001b[36mUnitsAPI.list_units\u001b[39m\u001b[34m(self, parent_id, level, page, page_size, max_pages, sort, lang, format, if_none_match, if_modified_since, extra_query)\u001b[39m\n\u001b[32m 94\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mlist_units\u001b[39m(\n\u001b[32m 95\u001b[39m \u001b[38;5;28mself\u001b[39m,\n\u001b[32m 96\u001b[39m parent_id: \u001b[38;5;28mstr\u001b[39m | \u001b[38;5;28;01mNone\u001b[39;00m = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m (...)\u001b[39m\u001b[32m 106\u001b[39m extra_query: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any] | \u001b[38;5;28;01mNone\u001b[39;00m = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m 107\u001b[39m ) -> \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, Any]]:\n\u001b[32m--> \u001b[39m\u001b[32m108\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_fetch_collection_endpoint\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 109\u001b[39m \u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43munits\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 110\u001b[39m \u001b[43m \u001b[49m\u001b[43mextra_params\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_list_units_params\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparent_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msort\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_query\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 111\u001b[39m \u001b[43m \u001b[49m\u001b[43mlang\u001b[49m\u001b[43m=\u001b[49m\u001b[43mlang\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 112\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mformat\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 113\u001b[39m \u001b[43m \u001b[49m\u001b[43mif_none_match\u001b[49m\u001b[43m=\u001b[49m\u001b[43mif_none_match\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 114\u001b[39m \u001b[43m \u001b[49m\u001b[43mif_modified_since\u001b[49m\u001b[43m=\u001b[49m\u001b[43mif_modified_since\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 115\u001b[39m \u001b[43m \u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 116\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 117\u001b[39m \u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mresults\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 118\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:738\u001b[39m, in \u001b[36mBaseAPIClient._fetch_collection_endpoint\u001b[39m\u001b[34m(self, endpoint, extra_params, lang, format, if_none_match, if_modified_since, page_size, max_pages, results_key)\u001b[39m\n\u001b[32m 730\u001b[39m params_with_page_size[\u001b[33m\"\u001b[39m\u001b[33mpage-size\u001b[39m\u001b[33m\"\u001b[39m] = page_size\n\u001b[32m 731\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m.fetch_single_result(\n\u001b[32m 732\u001b[39m endpoint,\n\u001b[32m 733\u001b[39m results_key=results_key,\n\u001b[32m 734\u001b[39m params=params_with_page_size,\n\u001b[32m 735\u001b[39m headers=headers,\n\u001b[32m 736\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m738\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mfetch_all_results\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 739\u001b[39m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 740\u001b[39m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m=\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 741\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 742\u001b[39m \u001b[43m \u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 743\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 744\u001b[39m \u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 745\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:536\u001b[39m, in \u001b[36mBaseAPIClient.fetch_all_results\u001b[39m\u001b[34m(self, endpoint, method, params, headers, results_key, page_size, max_pages, return_metadata, show_progress)\u001b[39m\n\u001b[32m 534\u001b[39m first_page = \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m 535\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m536\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpage\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_paginated_request_sync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 537\u001b[39m \u001b[43m \u001b[49m\u001b[43mendpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 538\u001b[39m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 539\u001b[39m \u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m=\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 540\u001b[39m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 541\u001b[39m \u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m=\u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 542\u001b[39m \u001b[43m \u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 543\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmax_pages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 544\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 545\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mresults_key\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mpage\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 546\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mraise\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mBDLResponseError\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43mf\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mResponse does not contain key \u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mresults_key\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[33;43m'\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpayload\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpage\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:451\u001b[39m, in \u001b[36mBaseAPIClient._paginated_request_sync\u001b[39m\u001b[34m(self, endpoint, method, params, headers, results_key, page_size, max_pages, return_all)\u001b[39m\n\u001b[32m 449\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m next_url:\n\u001b[32m 450\u001b[39m \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m451\u001b[39m resp = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_request_sync_url\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnext_url\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m=\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m=\u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 453\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m results_key \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m resp:\n\u001b[32m 454\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m BDLResponseError(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mResponse does not contain key \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresults_key\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m\"\u001b[39m, payload=resp)\n", + "\u001b[36mFile \u001b[39m\u001b[32m~/git/uep/pyLDB/pybdl/api/client.py:363\u001b[39m, in \u001b[36mBaseAPIClient._request_sync_url\u001b[39m\u001b[34m(self, url, method, params, headers)\u001b[39m\n\u001b[32m 361\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m retries_429 < \u001b[38;5;28mself\u001b[39m.config.http_429_max_retries:\n\u001b[32m 362\u001b[39m retries_429 += \u001b[32m1\u001b[39m\n\u001b[32m--> \u001b[39m\u001b[32m363\u001b[39m \u001b[43mtime\u001b[49m\u001b[43m.\u001b[49m\u001b[43msleep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_retry_delay_after_429\u001b[49m\u001b[43m(\u001b[49m\u001b[43mretries_429\u001b[49m\u001b[43m \u001b[49m\u001b[43m-\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 364\u001b[39m \u001b[38;5;28;01mcontinue\u001b[39;00m\n\u001b[32m 365\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._process_response(response)\n", + "\u001b[31mKeyboardInterrupt\u001b[39m: " + ] } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" + ], + "source": [ + "# Enrich data with attribute labels and unit details.\n", + "# 'attributes' adds attr_name / attr_symbol columns; 'units' adds unit_name_enriched, unit_level, etc.\n", + "data = bdl.data.get_data_by_variable(\n", + " variable_id=\"3643\",\n", + " years=[2021],\n", + " unit_level=2,\n", + " enrich=[\"attributes\", \"units\"],\n", + ")\n", + "print(\"Data with enriched attribute and unit columns:\")\n", + "print(data[[\"unit_id\", \"unit_name_enriched\", \"year\", \"val\", \"attr_id\", \"attr_name\"]].head())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Fetching 3643: 1 pages [00:00, 161.21 pages/s, items=16]\n", + "Fetching attributes: 1 pages [00:00, 15.45 pages/s, items=18]" + ] }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.0" + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total pages: None, Total records: 16\n", + " unit_id year val attr_name\n", + "0 11200000000 2021 2 value\n", + "1 12400000000 2021 6 value\n", + "2 20800000000 2021 0 0\n", + "3 23000000000 2021 2 value\n", + "4 23200000000 2021 5 value\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] } + ], + "source": [ + "# Combine enrichment with metadata retrieval.\n", + "# return_metadata=True returns a (DataFrame, metadata_dict) tuple.\n", + "df, meta = bdl.data.get_data_by_variable(\n", + " variable_id=\"3643\",\n", + " years=[2021],\n", + " unit_level=2,\n", + " enrich=[\"attributes\"],\n", + " return_metadata=True,\n", + ")\n", + "print(f\"Total pages: {meta.get('totalPages')}, Total records: {meta.get('totalRecords')}\")\n", + "print(df[[\"unit_id\", \"year\", \"val\", \"attr_name\"]].head())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Best Practices\n", + "\n", + "1. **Use the access layer**: Prefer ``bdl.data`` over ``bdl.api.data`` for DataFrame output\n", + "2. **Enable caching**: Use ``use_cache=True`` for repeated queries\n", + "3. **Handle pagination**: Use ``max_pages=None`` to get complete datasets\n", + "4. **Use enrichment**: Let the library automatically add human-readable names\n", + "5. **Async for bulk operations**: Use async methods when fetching multiple datasets\n", + "6. **Error handling**: Always handle ``RateLimitError`` and check for empty results\n", + "\n", + "```{seealso}\n", + "- {doc}`access_layer` for access layer documentation\n", + "- {doc}`rate_limiting` for rate limiting details\n", + "- {doc}`config` for configuration options\n", + "```\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 2 + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/docs/index.rst b/docs/index.rst index 7021812..8fe7330 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,14 +39,14 @@ Quick Start .. code-block:: python from pybdl import BDL, BDLConfig - + # Initialize client bdl = BDL(BDLConfig(api_key="your-api-key")) # Reads config from environment or defaults - + # Use the access layer (returns pandas DataFrames) df = bdl.data.get_data_by_variable(variable_id="3643", years=[2021]) print(df.head()) - + # Data is ready for analysis print(df.dtypes) print(df.columns) @@ -93,4 +93,3 @@ Contributing & License pyBDL is open source under the MIT license. Contributions and issues are welcome! For details, see the `GitHub repository `_. - diff --git a/docs/main_client.rst b/docs/main_client.rst index 58458ce..63c4ec6 100644 --- a/docs/main_client.rst +++ b/docs/main_client.rst @@ -23,14 +23,14 @@ The access layer is the primary interface and returns pandas DataFrames: .. code-block:: python from pybdl import BDL, BDLConfig - + bdl = BDL(BDLConfig(api_key="your-api-key")) - + # Access layer - returns DataFrames levels_df = bdl.levels.list_levels() variables_df = bdl.variables.list_variables() data_df = bdl.data.get_data_by_variable(variable_id="3643", years=[2021]) - + # Data is ready for pandas operations print(levels_df.head()) print(data_df.dtypes) @@ -53,13 +53,13 @@ The API layer provides direct access to raw API responses as dictionaries: .. code-block:: python from pybdl import BDL - + bdl = BDL() - + # API layer - returns raw dictionaries levels_data = bdl.api.levels.list_levels() data_dict = bdl.api.data.get_data_by_variable(variable_id="3643", years=[2021]) - + # Raw API response structure print(type(levels_data)) # list print(type(data_dict)) # list or dict @@ -82,18 +82,18 @@ Basic Usage with Access Layer .. code-block:: python from pybdl import BDL, BDLConfig - + # Initialize client bdl = BDL(BDLConfig(api_key="your-api-key")) - + # Get administrative levels levels = bdl.levels.list_levels() print(f"Found {len(levels)} administrative levels") - + # Get variables variables = bdl.variables.search_variables(name="population") print(f"Found {len(variables)} population variables") - + # Get data data = bdl.data.get_data_by_variable("3643", years=[2021], unit_level=2) print(f"Retrieved {len(data)} data points") @@ -105,13 +105,13 @@ Using Both Interfaces .. code-block:: python from pybdl import BDL - + bdl = BDL() - + # Access layer for DataFrame analysis df = bdl.data.get_data_by_variable("3643", years=[2021]) avg_value = df['val'].mean() - + # API layer for raw metadata metadata = bdl.api.data.get_data_by_variable( "3643", years=[2021], return_metadata=True @@ -169,19 +169,19 @@ Both interfaces support async operations: import asyncio from pybdl import BDL - + async def main(): bdl = BDL() - + # Async access layer levels_df = await bdl.levels.alist_levels() variables_df = await bdl.variables.alist_variables() - + # Async API layer levels_data = await bdl.api.levels.alist_levels() - + return levels_df, variables_df, levels_data - + asyncio.run(main()) .. seealso:: diff --git a/docs/rate_limiting.rst b/docs/rate_limiting.rst index b72beef..a2a8170 100644 --- a/docs/rate_limiting.rst +++ b/docs/rate_limiting.rst @@ -61,10 +61,10 @@ Rate limiting is automatically handled by the library. Simply use the API client .. code-block:: python from pybdl import BDL, BDLConfig - + config = BDLConfig(api_key="your-api-key") bdl = BDL(config) - + # Rate limiting is automatic - no extra code needed data = bdl.api.data.get_data_by_variable(variable_id="3643", years=[2021]) @@ -81,7 +81,7 @@ By default, the rate limiter raises a :class:`RateLimitError` when quota is exce .. code-block:: python from pybdl.utils.rate_limiter import RateLimitError - + try: data = bdl.api.data.get_data_by_variable(variable_id="3643", years=[2021]) except RateLimitError as e: @@ -101,7 +101,7 @@ You can configure the rate limiter to wait automatically instead of raising exce from pybdl.utils.rate_limiter import RateLimiter, PersistentQuotaCache from pybdl.config import DEFAULT_QUOTAS - + # Create a rate limiter that waits up to 30 seconds cache = PersistentQuotaCache(enabled=True) quotas = {k: v[1] for k, v in DEFAULT_QUOTAS.items()} # Registered user quotas @@ -112,7 +112,7 @@ You can configure the rate limiter to wait automatically instead of raising exce raise_on_limit=False, # Wait instead of raising max_delay=30.0 # Maximum wait time in seconds ) - + # Use the limiter before making API calls limiter.acquire() data = bdl.api.data.get_data_by_variable(variable_id="3643", years=[2021]) @@ -126,11 +126,11 @@ Rate limiters can be used as context managers for cleaner code: from pybdl.utils.rate_limiter import RateLimiter, PersistentQuotaCache from pybdl.config import DEFAULT_QUOTAS - + cache = PersistentQuotaCache(enabled=True) quotas = {k: v[1] for k, v in DEFAULT_QUOTAS.items()} limiter = RateLimiter(quotas, is_registered=True, cache=cache) - + # Automatically acquires quota when entering context with limiter: data = bdl.api.data.get_data_by_variable(variable_id="3643", years=[2021]) @@ -144,13 +144,13 @@ You can decorate functions to automatically rate limit them: from pybdl.utils.rate_limiter import rate_limit from pybdl.config import DEFAULT_QUOTAS - + quotas = {k: v[1] for k, v in DEFAULT_QUOTAS.items()} - + @rate_limit(quotas=quotas, is_registered=True, max_delay=10) def fetch_data(variable_id: str, year: int): return bdl.api.data.get_data_by_variable(variable_id=variable_id, years=[year]) - + # Function is automatically rate limited data = fetch_data("3643", 2021) @@ -160,9 +160,9 @@ For async functions: from pybdl.utils.rate_limiter import async_rate_limit from pybdl.config import DEFAULT_QUOTAS - + quotas = {k: v[1] for k, v in DEFAULT_QUOTAS.items()} - + @async_rate_limit(quotas=quotas, is_registered=True) async def async_fetch_data(variable_id: str, year: int): return await bdl.api.data.aget_data_by_variable(variable_id=variable_id, years=[year]) @@ -175,9 +175,9 @@ You can check how much quota remains before making API calls: .. code-block:: python from pybdl import BDL, BDLConfig - + bdl = BDL(BDLConfig(api_key="your-api-key")) - + # Get remaining quota (requires accessing the internal limiter) remaining = bdl._client._sync_limiter.get_remaining_quota() print(f"Remaining requests per second: {remaining.get(1, 0)}") @@ -191,7 +191,7 @@ You can override default quotas for testing or special deployments: .. code-block:: python from pybdl import BDLConfig - + # Custom quotas: period in seconds -> limit custom_quotas = { 1: 20, # 20 requests per second @@ -199,7 +199,7 @@ You can override default quotas for testing or special deployments: 43200: 2000, # 2000 requests per 12 hours 604800: 20000 # 20000 requests per 7 days } - + config = BDLConfig(api_key="your-api-key", custom_quotas=custom_quotas) bdl = BDL(config) @@ -222,7 +222,7 @@ You can disable persistent caching: .. code-block:: python from pybdl import BDLConfig - + config = BDLConfig(api_key="your-api-key", quota_cache_enabled=False) bdl = BDL(config) @@ -258,13 +258,13 @@ Example: Custom Rate Limiter with Wait Behavior from pybdl.utils.rate_limiter import RateLimiter, PersistentQuotaCache from pybdl.config import DEFAULT_QUOTAS - + # Create cache cache = PersistentQuotaCache(enabled=True) - + # Get registered user quotas quotas = {k: v[1] for k, v in DEFAULT_QUOTAS.items()} - + # Create limiter that waits up to 30 seconds limiter = RateLimiter( quotas=quotas, @@ -273,7 +273,7 @@ Example: Custom Rate Limiter with Wait Behavior raise_on_limit=False, max_delay=30.0 ) - + # Use limiter limiter.acquire() # Will wait if needed, up to 30 seconds # Make your API call here @@ -285,9 +285,9 @@ Example: Handling Rate Limit Errors from pybdl import BDL, BDLConfig from pybdl.utils.rate_limiter import RateLimitError, RateLimitDelayExceeded - + bdl = BDL(BDLConfig(api_key="your-api-key")) - + try: data = bdl.api.data.get_data_by_variable(variable_id="3643", years=[2021]) except RateLimitError as e: @@ -303,16 +303,16 @@ Example: Checking Quota Before Making Calls .. code-block:: python from pybdl import BDL, BDLConfig - + bdl = BDL(BDLConfig(api_key="your-api-key")) - + # Check remaining quota remaining = bdl._client._sync_limiter.get_remaining_quota() - + if remaining.get(1, 0) < 5: print("Warning: Low quota remaining for 1-second period") # Consider waiting or reducing request rate - + # Make API call data = bdl.api.data.get_data_by_variable(variable_id="3643", years=[2021]) @@ -322,12 +322,12 @@ Example: Resetting Quota (for testing) .. code-block:: python from pybdl import BDL, BDLConfig - + bdl = BDL(BDLConfig(api_key="your-api-key")) - + # Reset quota counters (useful for testing) bdl._client._sync_limiter.reset() - + # Now you can make fresh API calls Best Practices @@ -363,4 +363,3 @@ A: The cache file is automatically recreated if corrupted. Old quota data will b - :doc:`config` for configuration options - :doc:`api_clients` for API usage examples - :doc:`appendix` for technical implementation details - diff --git a/git-conventional-commits.yaml b/git-conventional-commits.yaml index 3092f0d..2fde4af 100644 --- a/git-conventional-commits.yaml +++ b/git-conventional-commits.yaml @@ -12,7 +12,7 @@ convention: - perf - ci - merge - commitScopes: [ ] + commitScopes: [] releaseTagGlobPattern: v[0-9]*.[0-9]*.[0-9]* changelog: commitTypes: @@ -31,4 +31,4 @@ changelog: # GitHub commitUrl: https://github.com/AN0DA/pybdl/commit/%commit% - commitRangeUrl: https://github.com/AN0DA/pybdl/compare/%from%...%to%?diff=split \ No newline at end of file + commitRangeUrl: https://github.com/AN0DA/pybdl/compare/%from%...%to%?diff=split diff --git a/pyproject.toml b/pyproject.toml index 0e44973..1743a7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dev = [ "bandit>=1.8.3", "mypy>=1.15.0", "notebook>=7.4.3", + "prek>=0.3.8", "pytest-asyncio>=1.0.0", "pytest-cov>=6.0.0", "pytest-mock>=3.14.0", @@ -70,6 +71,7 @@ exclude = [ "buck-out", "build", "dist", + "docs", "node_modules", "site-packages", "venv", @@ -148,6 +150,9 @@ warn_return_any = false ignore_missing_imports = true disable_error_code = "abstract" +[[tool.mypy.overrides]] +module = "tests.*" +disable_error_code = ["untyped-decorator"] [tool.bandit] diff --git a/tests/integration/access/samples/raw/samples_raw_data.json b/tests/integration/access/samples/raw/samples_raw_data.json index aa9775b..11fb469 100644 --- a/tests/integration/access/samples/raw/samples_raw_data.json +++ b/tests/integration/access/samples/raw/samples_raw_data.json @@ -634,6 +634,6 @@ } ] }, - "get_data_by_variable_locality": [], + "get_data_by_variable_locality": [], "get_data_by_unit_locality": [] } diff --git a/tests/integration/access/samples/raw/samples_raw_variables.json b/tests/integration/access/samples/raw/samples_raw_variables.json index d91a638..15e639a 100644 --- a/tests/integration/access/samples/raw/samples_raw_variables.json +++ b/tests/integration/access/samples/raw/samples_raw_variables.json @@ -217,4 +217,4 @@ "measureUnitId": 26, "measureUnitName": "osoba" } - ]} \ No newline at end of file + ]} diff --git a/tests/integration/access/samples/raw/samples_raw_years.json b/tests/integration/access/samples/raw/samples_raw_years.json index 525f630..ded9448 100644 --- a/tests/integration/access/samples/raw/samples_raw_years.json +++ b/tests/integration/access/samples/raw/samples_raw_years.json @@ -161,4 +161,4 @@ "hasLocalities": true, "quarterly": "M" } -} \ No newline at end of file +} diff --git a/uv.lock b/uv.lock index cbb2529..b2782b0 100644 --- a/uv.lock +++ b/uv.lock @@ -2140,6 +2140,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "prek" +version = "0.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/62/ee/03e8180e3fda9de25b6480bd15cc2bde40d573868d50648b0e527b35562f/prek-0.3.8.tar.gz", hash = "sha256:434a214256516f187a3ab15f869d950243be66b94ad47987ee4281b69643a2d9", size = 400224, upload-time = "2026-03-23T08:23:35.981Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/84/40d2ddf362d12c4cd4a25a8c89a862edf87cdfbf1422aa41aac8e315d409/prek-0.3.8-py3-none-linux_armv6l.whl", hash = "sha256:6fb646ada60658fa6dd7771b2e0fb097f005151be222f869dada3eb26d79ed33", size = 5226646, upload-time = "2026-03-23T08:23:18.306Z" }, + { url = "https://files.pythonhosted.org/packages/e1/52/7308a033fa43b7e8e188797bd2b3b017c0f0adda70fa7af575b1f43ea888/prek-0.3.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3d7fdadb15efc19c09953c7a33cf2061a70f367d1e1957358d3ad5cc49d0616", size = 5620104, upload-time = "2026-03-23T08:23:40.053Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b1/f106ac000a91511a9cd80169868daf2f5b693480ef5232cec5517a38a512/prek-0.3.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:72728c3295e79ca443f8c1ec037d2a5b914ec73a358f69cf1bc1964511876bf8", size = 5199867, upload-time = "2026-03-23T08:23:38.066Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e9/970713f4b019f69de9844e1bab37b8ddb67558e410916f4eb5869a696165/prek-0.3.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:48efc28f2f53b5b8087efca9daaed91572d62df97d5f24a1c7a087fecb5017de", size = 5441801, upload-time = "2026-03-23T08:23:32.617Z" }, + { url = "https://files.pythonhosted.org/packages/12/a4/7ef44032b181753e19452ec3b09abb3a32607cf6b0a0508f0604becaaf2b/prek-0.3.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6ca9d63bacbc448a5c18e955c78d3ac5176c3a17c3baacdd949b1a623e08a36", size = 5155107, upload-time = "2026-03-23T08:23:31.021Z" }, + { url = "https://files.pythonhosted.org/packages/bd/77/4d9c8985dbba84149760785dfe07093ea1e29d710257dfb7c89615e2234c/prek-0.3.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1000f7029696b4fe712fb1fefd4c55b9c4de72b65509c8e50296370a06f9dc3f", size = 5566541, upload-time = "2026-03-23T08:23:45.694Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1a/81e6769ac1f7f8346d09ce2ab0b47cf06466acd9ff72e87e5d1f0d98cd32/prek-0.3.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ff0bed0e2c1286522987d982168a86cbbd0d069d840506a46c9fda983515517", size = 6552991, upload-time = "2026-03-23T08:23:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/6f/fa/ce2df0dd2dc75a9437a52463239d0782998943d7b04e191fb89b83016c34/prek-0.3.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fb087ac0ffda3ac65bbbae9a38326a7fd27ee007bb4a94323ce1eb539d8bbec", size = 5832972, upload-time = "2026-03-23T08:23:20.258Z" }, + { url = "https://files.pythonhosted.org/packages/18/6b/9d4269df9073216d296244595a21c253b6475dfc9076c0bd2906be7a436c/prek-0.3.8-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:2e1e5e206ff7b31bd079cce525daddc96cd6bc544d20dc128921ad92f7a4c85d", size = 5448371, upload-time = "2026-03-23T08:23:41.835Z" }, + { url = "https://files.pythonhosted.org/packages/60/1d/1e4d8a78abefa5b9d086e5a9f1638a74b5e540eec8a648d9946707701f29/prek-0.3.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:dcea3fe23832a4481bccb7c45f55650cb233be7c805602e788bb7dba60f2d861", size = 5270546, upload-time = "2026-03-23T08:23:24.231Z" }, + { url = "https://files.pythonhosted.org/packages/77/07/34f36551a6319ae36e272bea63a42f59d41d2d47ab0d5fb00eb7b4e88e87/prek-0.3.8-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:4d25e647e9682f6818ab5c31e7a4b842993c14782a6ffcd128d22b784e0d677f", size = 5124032, upload-time = "2026-03-23T08:23:26.368Z" }, + { url = "https://files.pythonhosted.org/packages/e3/01/6d544009bb655e709993411796af77339f439526db4f3b3509c583ad8eb9/prek-0.3.8-py3-none-musllinux_1_1_i686.whl", hash = "sha256:de528b82935e33074815acff3c7c86026754d1212136295bc88fe9c43b4231d5", size = 5432245, upload-time = "2026-03-23T08:23:47.877Z" }, + { url = "https://files.pythonhosted.org/packages/54/96/1237ee269e9bfa283ffadbcba1f401f48a47aed2b2563eb1002740d6079d/prek-0.3.8-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:6d660f1c25a126e6d9f682fe61449441226514f412a4469f5d71f8f8cad56db2", size = 5950550, upload-time = "2026-03-23T08:23:43.8Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6b/a574411459049bc691047c9912f375deda10c44a707b6ce98df2b658f0b3/prek-0.3.8-py3-none-win32.whl", hash = "sha256:b0c291c577615d9f8450421dff0b32bfd77a6b0d223ee4115a1f820cb636fdf1", size = 4949501, upload-time = "2026-03-23T08:23:16.338Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b4/46b59fe49f635acd9f6530778ce577f9d8b49452835726a5311ffc902c67/prek-0.3.8-py3-none-win_amd64.whl", hash = "sha256:bc147fdbdd4ec33fc7a987b893ecb69b1413ac100d95c9889a70f3fd58c73d06", size = 5346551, upload-time = "2026-03-23T08:23:34.501Z" }, + { url = "https://files.pythonhosted.org/packages/53/05/9cca1708bb8c65264124eb4b04251e0f65ce5bfc707080bb6b492d5a0df7/prek-0.3.8-py3-none-win_arm64.whl", hash = "sha256:a2614647aeafa817a5802ccb9561e92eedc20dcf840639a1b00826e2c2442515", size = 5190872, upload-time = "2026-03-23T08:23:29.463Z" }, +] + [[package]] name = "prometheus-client" version = "0.24.1" @@ -2231,6 +2255,7 @@ dev = [ { name = "bandit" }, { name = "mypy" }, { name = "notebook" }, + { name = "prek" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -2266,6 +2291,7 @@ dev = [ { name = "bandit", specifier = ">=1.8.3" }, { name = "mypy", specifier = ">=1.15.0" }, { name = "notebook", specifier = ">=7.4.3" }, + { name = "prek", specifier = ">=0.3.8" }, { name = "pytest", specifier = ">=8.3.5" }, { name = "pytest-asyncio", specifier = ">=1.0.0" }, { name = "pytest-cov", specifier = ">=6.0.0" },