diff --git a/.azuredevops/ado-ci-pipeline-ms-hosted.yml b/.azuredevops/ado-ci-pipeline-ms-hosted.yml index 3ca1a69..58ffedb 100644 --- a/.azuredevops/ado-ci-pipeline-ms-hosted.yml +++ b/.azuredevops/ado-ci-pipeline-ms-hosted.yml @@ -1,62 +1,81 @@ -# Azure DevOps pipeline for CI (Microsoft-hosted version) -# As the Microsoft-hosted agent option has a limit of 10GB of storage for disk outputs from a pipeline, -# this causes an issue when the Docker images for modules under src require more than 10GB of storage. -# If you will run into space issues (or other limitations with a Microsoft hosted agent option outlined in -# https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#capabilities-and-limitations), -# consider using the .azuredevops/ado-ci-pipeline-self-hosted.yml version or using scale set agents, see -# this link for more info: https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/scale-set-agents?view=azure-devops -# Note that docker images will only be build for src directories that contain at least one test file, so the -# total space consumed by Docker builds will be dependent on which modules under src contain tests. -# For setting up the pipeline in ADO see: -# https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/pools-queues?view=azure-devops&tabs=yaml%2Cbrowser - - -trigger: - - main - -pool: - vmImage: 'ubuntu-latest' - -steps: - - task: UsePythonVersion@0 - displayName: "Use Python 3.11" - inputs: - versionSpec: 3.11 - - - script: | - python -m venv venv - source venv/bin/activate - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - pip install pytest-azurepipelines - displayName: "Install requirements" - - # files under venv will be automatically excluded from ruff check by default https://docs.astral.sh/ruff/settings/#exclude - - bash: | - source venv/bin/activate - ruff check --output-format azure - displayName: "Run ruff linter" - - - task: Bash@3 - inputs: - targetType: 'filePath' - filePath: ci-tests.sh - env: - BUILD_ARTIFACTSTAGINGDIRECTORY: $(Build.ArtifactStagingDirectory) - displayName: "Run pytest in docker containers" - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results-*.xml' - searchFolder: $(Build.ArtifactStagingDirectory) - condition: succeededOrFailed() - - # Publish code coverage results - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: 'Cobertura' # Available options: 'JaCoCo', 'Cobertura' - summaryFileLocation: '$(Build.ArtifactStagingDirectory)/coverage.xml' - pathToSources: src/ - #reportDirectory: # Optional - #additionalCodeCoverageFiles: # Optional - failIfCoverageEmpty: false # Optional +# Azure DevOps pipeline for CI (Microsoft-hosted version) +# As the Microsoft-hosted agent option has a limit of 10GB of storage for disk outputs from a pipeline, +# this causes an issue when the Docker images for modules under src require more than 10GB of storage. +# If you will run into space issues (or other limitations with a Microsoft hosted agent option outlined in +# https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#capabilities-and-limitations), +# consider using the .azuredevops/ado-ci-pipeline-self-hosted.yml version or using scale set agents, see +# this link for more info: https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/scale-set-agents?view=azure-devops +# Note that docker images will only be build for src directories that contain at least one test file, so the +# total space consumed by Docker builds will be dependent on which modules under src contain tests. +# For setting up the pipeline in ADO see: +# https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/pools-queues?view=azure-devops&tabs=yaml%2Cbrowser +# +# Templates are located in .azuredevops/templates/ + +trigger: + - main + +stages: + - stage: Lint + displayName: 'Lint' + jobs: + - job: RunLinter + displayName: 'Run ruff linter' + pool: + vmImage: 'ubuntu-latest' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: 3.11 + + - script: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + displayName: 'Install requirements' + + - bash: | + source venv/bin/activate + ruff check --output-format azure + displayName: 'Run ruff linter' + + - stage: Test + displayName: 'Test Devcontainers' + jobs: + - template: templates/test-devcontainer-job.yml + parameters: + projects: + - name: 'cpu-project' + configPath: 'src/sample_cpu_project/.devcontainer/devcontainer.json' + projectPath: 'src/sample_cpu_project' + - name: 'gpu-project' + configPath: 'src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json' + projectPath: 'src/sample_pytorch_gpu_project' + - name: 'notebooks' + configPath: 'notebooks/.devcontainer/devcontainer.json' + projectPath: 'notebooks' + smokeTestOnly: true + pool: + vmImage: 'ubuntu-latest' + + - job: PublishCoverage + displayName: 'Publish Coverage' + # NOTE: test-devcontainer-job.yml creates one job per project with the name pattern: + # Test_ + # For the projects defined above: + # - cpu-project -> Test_cpu_project + # - gpu-project -> Test_gpu_project + # - notebooks -> Test_notebooks + # If you change a project name, update the corresponding entry here to match this convention. + dependsOn: [Test_cpu_project, Test_gpu_project, Test_notebooks] + condition: succeededOrFailed() + pool: + vmImage: 'ubuntu-latest' + steps: + - template: templates/merge-coverage.yml + parameters: + coverageArtifacts: + - 'coverage-cpu-project' + - 'coverage-gpu-project' diff --git a/.azuredevops/ado-ci-pipeline-self-hosted.yml b/.azuredevops/ado-ci-pipeline-self-hosted.yml index 06b58bd..c264f26 100644 --- a/.azuredevops/ado-ci-pipeline-self-hosted.yml +++ b/.azuredevops/ado-ci-pipeline-self-hosted.yml @@ -1,82 +1,62 @@ -# Azure DevOps pipeline for CI (self-hoseted version) -# As the Microsoft-hosted agent option has a limit of 10GB of storage for disk outputs from a pipeline, -# this causes an issue when the Docker images for modules under src require more than 10GB of storage. -# The self-hosted agent option allows the storage to be increased based on the VM size. This version -# includes extra clean-up and space management steps relating to docker builds, but it otherwise equivalent -# to the .azuredevops/ado-ci-pipeline-ms-hosted.yml version. -# For setting up a CI pipeline with a self-hosted Linux agent see: -# https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/v2-linux?view=azure-devops -# Note that the CI scripts that this pipeline runs (ci-tests.sh) is designed to be run on a Linux agent, -# but could be adapated to other OSs. - +# Azure DevOps CI Pipeline (Self-Hosted Agent) +# +# This pipeline runs on self-hosted agents and supports both Linux and macOS (Apple Silicon). +# Templates are located in .azuredevops/templates/ trigger: - main -pool: - name: Default - demands: - - agent.name -equals mc-ubuntu-agent -workspace: - clean: all - -steps: - - script: | - docker image prune -f - docker container prune -f - displayName: "Docker Cleanup" - - - script: | - df -h - displayName: "Check agent VM space" - - - task: UsePythonVersion@0 - displayName: "Use Python 3.11" - inputs: - versionSpec: 3.11 - - - script: | - python -m venv venv - source venv/bin/activate - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - pip install pytest-azurepipelines - displayName: "Install requirements" - - - task: UseDotNet@2 - inputs: - packageType: 'sdk' - workingDirectory: "src/" - version: '6.x' - - # files under venv will be automatically excluded from ruff check by default https://docs.astral.sh/ruff/settings/#exclude - - bash: | - source venv/bin/activate - ruff check --output-format azure - displayName: "Run ruff linter" - - - task: Bash@3 - inputs: - targetType: 'filePath' - filePath: ci-tests.sh - displayName: "Run pytest in docker containers" - - - task: PublishTestResults@2 - inputs: - testResultsFiles: "/tmp/artifact_output/**/test-results-*.xml" - condition: succeededOrFailed() - - # Publish code coverage results - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: 'Cobertura' # Available options: 'JaCoCo', 'Cobertura' - summaryFileLocation: '/tmp/artifact_output/coverage.xml' - pathToSources: src/ - #reportDirectory: # Optional - #additionalCodeCoverageFiles: # Optional - failIfCoverageEmpty: false # Optional - - - bash: | - sudo rm -rfv /home/azureuser/myagent/_work/* /home/azureuser/myagent/_work/.* || true - displayName: "Clean-up _work dir" - condition: always() +stages: + - stage: Lint + displayName: 'Lint' + jobs: + - job: RunLinter + displayName: 'Run ruff linter' + pool: + name: Default + workspace: + clean: all + steps: + - script: | + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install -r requirements-dev.txt + displayName: 'Install requirements' + + - bash: | + source venv/bin/activate + ruff check --output-format azure + displayName: 'Run ruff linter' + + - stage: Test + displayName: 'Test Devcontainers' + jobs: + - template: templates/test-devcontainer-job.yml + parameters: + projects: + - name: 'cpu-project' + configPath: 'src/sample_cpu_project/.devcontainer/devcontainer.json' + projectPath: 'src/sample_cpu_project' + - name: 'gpu-project' + configPath: 'src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json' + projectPath: 'src/sample_pytorch_gpu_project' + - name: 'notebooks' + configPath: 'notebooks/.devcontainer/devcontainer.json' + projectPath: 'notebooks' + smokeTestOnly: true + pool: + name: Default + + - job: PublishCoverage + displayName: 'Publish Coverage' + dependsOn: [Test_cpu_project, Test_gpu_project, Test_notebooks] + condition: succeededOrFailed() + pool: + name: Default + steps: + - template: templates/merge-coverage.yml + parameters: + coverageArtifacts: + - 'coverage-cpu-project' + - 'coverage-gpu-project' diff --git a/.azuredevops/templates/merge-coverage.yml b/.azuredevops/templates/merge-coverage.yml new file mode 100644 index 0000000..89e1bfc --- /dev/null +++ b/.azuredevops/templates/merge-coverage.yml @@ -0,0 +1,55 @@ +parameters: + - name: coverageArtifacts + type: object + default: [] + +steps: + - checkout: none + + - ${{ each artifact in parameters.coverageArtifacts }}: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: '${{ artifact }}' + path: '$(Pipeline.Workspace)/coverage/${{ artifact }}' + continueOnError: true + + - bash: | + set -ex + + # Install .NET SDK + ARCH_FLAG="" + [[ "$(uname -m)" == "arm64" ]] && ARCH_FLAG="--architecture arm64" + curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin --channel 8.0 --install-dir $HOME/.dotnet $ARCH_FLAG + + # Make .NET available for subsequent tasks + echo "##vso[task.prependpath]$HOME/.dotnet" + echo "##vso[task.prependpath]$HOME/.dotnet/tools" + displayName: 'Install .NET SDK' + + - bash: | + set -ex + + export DOTNET_ROOT=$HOME/.dotnet + export PATH="$DOTNET_ROOT:$DOTNET_ROOT/tools:$PATH" + + # Install ReportGenerator + dotnet tool install -g dotnet-reportgenerator-globaltool 2>/dev/null || dotnet tool update -g dotnet-reportgenerator-globaltool + + # Find and merge coverage files + COVERAGE_FILES=$(find $(Pipeline.Workspace)/coverage -name "coverage.xml" 2>/dev/null | paste -sd ";" -) + if [ -z "$COVERAGE_FILES" ]; then + echo "##vso[task.logissue type=error]No coverage files found" + exit 1 + fi + + echo "Merging coverage files: $COVERAGE_FILES" + mkdir -p $(Pipeline.Workspace)/coverage/merged + reportgenerator \ + "-reports:$COVERAGE_FILES" \ + "-targetdir:$(Pipeline.Workspace)/coverage/merged" \ + "-reporttypes:Cobertura;HtmlInline_AzurePipelines" + displayName: 'Merge coverage reports' + + - task: PublishCodeCoverageResults@2 + inputs: + summaryFileLocation: '$(Pipeline.Workspace)/coverage/merged/Cobertura.xml' diff --git a/.azuredevops/templates/publish-test-results.yml b/.azuredevops/templates/publish-test-results.yml new file mode 100644 index 0000000..6622634 --- /dev/null +++ b/.azuredevops/templates/publish-test-results.yml @@ -0,0 +1,25 @@ +# Template: Publish Test Results +# Publishes test results and coverage artifacts from a project +# +# Parameters: +# projectPath: Path to project directory containing test-results.xml and coverage.xml +# projectName: Name for the test run and coverage artifact + +parameters: + - name: projectPath + type: string + - name: projectName + type: string + +steps: + - task: PublishTestResults@2 + inputs: + testResultsFiles: '${{ parameters.projectPath }}/test-results.xml' + testRunTitle: '${{ parameters.projectName }}' + condition: succeededOrFailed() + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(System.DefaultWorkingDirectory)/${{ parameters.projectPath }}/coverage.xml' + artifact: 'coverage-${{ parameters.projectName }}' + condition: succeededOrFailed() diff --git a/.azuredevops/templates/run-devcontainer.yml b/.azuredevops/templates/run-devcontainer.yml new file mode 100644 index 0000000..b9c2419 --- /dev/null +++ b/.azuredevops/templates/run-devcontainer.yml @@ -0,0 +1,57 @@ +# Template: Run Devcontainer +# Builds and executes commands inside a devcontainer +# +# Parameters: +# configPath: Path to devcontainer.json relative to repo root +# command: Shell command to execute inside the container +# +# Handles platform-specific Docker configuration: +# - macOS: Uses DOCKER_HOST to avoid Docker Desktop path translation issues +# - Linux: Uses standard docker socket + +parameters: + - name: configPath + type: string + - name: command + type: string + +steps: + - bash: | + set -ex + + if [[ "$OSTYPE" == "darwin"* ]]; then + # Use unix socket directly to bypass Docker Desktop context issues on macOS + export DOCKER_HOST=unix:///var/run/docker.sock + DOCKER_PATH=$(which docker) + + # Remove any existing container to force recreation with correct paths + docker ps -aq --filter "label=devcontainer.config_file=\"$(pwd)/${{ parameters.configPath }}\"" | xargs -r docker rm -f 2>/dev/null || true + docker ps -aq --filter "label=devcontainer.local_folder=\"$(pwd)\"" | xargs -r docker rm -f 2>/dev/null || true + + WORKSPACE_FOLDER="$(pwd)" + CONFIG_PATH="$WORKSPACE_FOLDER/${{ parameters.configPath }}" + + devcontainer up \ + --workspace-folder "$WORKSPACE_FOLDER" \ + --config "$CONFIG_PATH" \ + --docker-path "$DOCKER_PATH" + + # Run exec from /tmp to avoid Docker path translation issues on macOS + (cd /tmp && devcontainer exec \ + --workspace-folder "$WORKSPACE_FOLDER" \ + --config "$CONFIG_PATH" \ + --docker-path "$DOCKER_PATH" \ + bash -c '${{ parameters.command }}') + else + # Remove any existing container to ensure clean state on Linux + WORKSPACE_FOLDER="$(pwd)" + CONFIG_PATH="$WORKSPACE_FOLDER/${{ parameters.configPath }}" + docker ps -aq --filter "label=devcontainer.config_file=\"$CONFIG_PATH\"" | xargs -r docker rm -f 2>/dev/null || true + docker ps -aq --filter "label=devcontainer.local_folder=\"$WORKSPACE_FOLDER\"" | xargs -r docker rm -f 2>/dev/null || true + + devcontainer up --workspace-folder "$WORKSPACE_FOLDER" --config "$CONFIG_PATH" + # Run exec from /tmp to avoid any path issues + (cd /tmp && devcontainer exec --workspace-folder "$WORKSPACE_FOLDER" --config "$CONFIG_PATH" \ + bash -c '${{ parameters.command }}') + fi + displayName: 'Run devcontainer' diff --git a/.azuredevops/templates/setup-devcontainer.yml b/.azuredevops/templates/setup-devcontainer.yml new file mode 100644 index 0000000..4fbcb50 --- /dev/null +++ b/.azuredevops/templates/setup-devcontainer.yml @@ -0,0 +1,6 @@ +steps: + - script: cp .env.example .env + displayName: 'Setup .env file' + + - script: npm install -g @devcontainers/cli + displayName: 'Install devcontainer CLI' diff --git a/.azuredevops/templates/test-devcontainer-job.yml b/.azuredevops/templates/test-devcontainer-job.yml new file mode 100644 index 0000000..f10d26f --- /dev/null +++ b/.azuredevops/templates/test-devcontainer-job.yml @@ -0,0 +1,43 @@ +parameters: + - name: projects + type: object + - name: pool + type: object + +jobs: + - ${{ each project in parameters.projects }}: + - job: Test_${{ replace(project.name, '-', '_') }} + displayName: 'Test ${{ project.name }}' + pool: ${{ parameters.pool }} + workspace: + clean: all + steps: + - template: setup-devcontainer.yml + + - ${{ if eq(project.smokeTestOnly, true) }}: + - template: run-devcontainer.yml + parameters: + configPath: '${{ project.configPath }}' + command: 'cd /workspaces/repo && python --version && pytest --version && ruff --version' + + - ${{ if ne(project.smokeTestOnly, true) }}: + - template: run-devcontainer.yml + parameters: + configPath: '${{ project.configPath }}' + command: >- + cd /workspaces/repo/${{ project.projectPath }} && + pytest tests/ + --junitxml=/tmp/test-results.xml + -o junit_suite_name=${{ project.name }} + --doctest-modules + --cov=. + --cov-config=/workspaces/repo/pyproject.toml + --cov-report=xml:/tmp/coverage.xml && + sed -i "s|> $GITHUB_STEP_SUMMARY - - name: Archive test and code coverage results - uses: actions/upload-artifact@v5 + # Run linter on codebase + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python 3.11 + uses: actions/setup-python@v5 with: - name: test-and-coverage-results - path: | - **/test-reuslts-*.xml - coverage.xml + python-version: 3.11 + + - name: Install requirements + run: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Run ruff linter + run: | + source venv/bin/activate + ruff check --output-format github diff --git a/.github/workflows/sync-to-azdo.yaml b/.github/workflows/sync-to-azdo.yaml new file mode 100644 index 0000000..281f073 --- /dev/null +++ b/.github/workflows/sync-to-azdo.yaml @@ -0,0 +1,43 @@ +# GitHub Action to sync repository to Azure DevOps +# Automatically pushes commits from GitHub to Azure DevOps Repos +# This keeps Azure DevOps in sync when GitHub is the primary development location + +name: Sync to Azure DevOps + +on: + push: + branches: + - main + - feature/devcontainer-pytest-integration # Temporary for testing + workflow_dispatch: # Allow manual trigger + +jobs: + sync: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 # Fetch all history for complete mirror + + - name: Configure Git + run: | + git config --global user.email "github-actions@github.com" + git config --global user.name "GitHub Actions" + + - name: Push to Azure DevOps + env: + AZDO_PAT: ${{ secrets.AZDO_PAT }} + AZDO_ORG: ${{ secrets.AZDO_ORG }} + AZDO_PROJECT: ${{ secrets.AZDO_PROJECT }} + AZDO_REPO: ${{ secrets.AZDO_REPO }} + run: | + # Add Azure DevOps as remote + git remote add azdo https://${AZDO_PAT}@dev.azure.com/${AZDO_ORG}/${AZDO_PROJECT}/_git/${AZDO_REPO} + + # Push all branches and tags to Azure DevOps + git push azdo --all --force + git push azdo --tags --force + + echo "Successfully synced to Azure DevOps" diff --git a/.gitignore b/.gitignore index ad80d94..24429f3 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,12 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Speckit / .specify related files +.specify/ +specs/ +*.speckit + +# .github agents and prompts +.github/agents/ +.github/prompts/ diff --git a/README.md b/README.md index 6eeb6a1..5c413c7 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ A machine learning and data science project template that makes it easy to work - [`notebooks` directory vs `src` directory](#notebooks-directory-vs-src-directory) - [AML Example](#aml-example) - [CI Pipeline](#ci-pipeline) - - [Running all unit tests with `ci-tests.sh`](#running-all-unit-tests-with-ci-testssh) - [How to Configure Azure DevOps CI Pipeline](#how-to-configure-azure-devops-ci-pipeline) - [Choosing between Azure DevOps Microsoft-hosted vs Self-hosted CI Pipeline](#choosing-between-azure-devops-microsoft-hosted-vs-self-hosted-ci-pipeline) - [How to Configure Github Actions CI Pipeline](#how-to-configure-github-actions-ci-pipeline) @@ -73,12 +72,11 @@ This section gives you overview of the directory structure of this template. Onl ```bash . -├── .azuredevops # CI pipelines for Azure DevOps. Details at section: How to Configure Azure DevOps CI Pipeline -├── .github # CI pipelines for Github Actions. Details at section: How to Configure Github Actions CI Pipeline +├── .azuredevops # CI pipelines for Azure DevOps. Details at section: How to Configure Azure DevOps CI Pipeline +├── .github # CI pipelines for Github Actions. Details at section: How to Configure Github Actions CI Pipeline ├── .pre-commit-config.yaml # pre-commit config file with formatting and linting. Setup is covered in Section: Getting Started ├── .env.example # Example of .env file. Setup is covered in Section: Getting Started -├── ci-tests.sh # Details at Section: Running all unit tests with ci-tests.sh -├── data # Directory to keep your data for local training etc. This directory is gitignored +├── data # Directory to keep your data for local training etc. This directory is gitignored ├── notebooks # Setup process is covered in Section: How to setup dev environment? │ ├── .devcontainer # dev container related configuration files goes to here following VSCode convention │ │ ├── devcontainer.json # dev container configuration and VS Code settings, extensions etc. @@ -94,7 +92,7 @@ This section gives you overview of the directory structure of this template. Onl │ │ ├── devcontainer.json # dev container configuration and VS Code settings, extensions etc. │ │ ├── Dockerfile # referred in devcontainer.json. Supports only CPU │ │ └── requirements.txt # includes python package list for sample_cpu_project. used in Dockerfile - │ ├── sample_main.py + │ ├── sample_main.py │ └── tests # pytest scripts for sample_cpu_project goes here │ └── test_dummy.py # pytest script example └── sample_pytorch_gpu_project # gpu project example with pytorch. Setup process is covered in Section: How to setup dev environment? @@ -104,7 +102,7 @@ This section gives you overview of the directory structure of this template. Onl │ ├── Dockerfile # referred in devcontainer.json. Supports GPU │ └── requirements.txt # includes python package list for sample_pytorch_gpu_project. used in Dockerfile ├── aml_example/ # Sample AML CLI v2 Components-based pipeline, including setup YAML. See sample_pytorch_gpu_project/README for full details of files in this directory. - ├── sample_main.py + ├── sample_main.py ├── inference.py # Example pytorch inference/eval script that also works with aml_example ├── train.py # Example pytorch model training script that also works with aml_example └── tests # pytest scripts for sample_pytorch_gpu_project goes here @@ -132,24 +130,13 @@ An Azure Machine Learning (AML) example is provided under `src/sample_pytorch_gp This repository contains templates for running a Continuous Integration (CI) pipeline on either Azure DevOps (under `.azuredevops` directory) or on Github Actions (under `.github` directory). Each of the CI pipeline configurations provided have the following features at a high level: - Run code quality checks (`ruff check`) over the repository -- Find all subdirectories under `src` and run all pytest tests inside the associated Docker containers +- Build devcontainers and run pytest tests inside them using the devcontainer CLI (both GitHub Actions and Azure DevOps) - Publish test results and code coverage statistics We recommend setting up pipeline triggers for PR creation, editing and merging. This will ensure the pipeline runs continuously and will help catch any issues earlier in your development process. See the sections below for links on how to setup pipelines with [Azure DevOps](#how-to-configure-azure-devops-ci-pipeline) and [Github Actions](#how-to-configure-github-actions-ci-pipeline). Note that if you are only using one of these platforms to host a pipeline (or neither), you can safely delete either (or both) the `.azuredevops` directory or the `.github` directory. -### Running all unit tests with `ci-tests.sh` - -As multiple independent directories can be added under `src`, each with its own Dockerfile and requirements, running unit tests for each directory under `src` needs to be done within the Docker container of each `src` subdirectory. The `ci-tests.sh` script automates this task of running all unit tests for the repository with the following steps: - -1. Finds all subdirectories under `src` that have at least one `test_*.py` under a `tests` folder -2. Builds each Docker image for each subdirectory with tests, using the Dockerfile in the associated `.devcontainer` directory -3. Runs pytest for each subdirectory with tests, inside the matching Docker container built in step 2 -4. Combine all test results and coverage reports from step 3, with reports in a valid format for publishing in either Azure DevOps or Github Actions hosted pipeline - -Note that the `ci-test.sh` script can be run locally as well and it is assumed that all tests are written with pytest. - ### How to Configure Azure DevOps CI Pipeline See [create your first pipeline](https://learn.microsoft.com/en-us/azure/devops/pipelines/create-first-pipeline?view=azure-devops) for how to setup a pipeline in Azure DevOps. Note that to use the provided template in this repository, you will need to specify the path to `.azuredevops/ado-ci-pipeline-ms-hosted.yml` during the pipeline setup process in Azure DevOps. @@ -158,7 +145,7 @@ See [create your first pipeline](https://learn.microsoft.com/en-us/azure/devops/ There are two templates for running a CI pipeline in Azure DevOps, a pipeline configuration that uses a Microsoft hosted agent to run the pipeline (`.azuredevops/ado-ci-pipeline-ms-hosted.yml`) and a pipeline configuration that uses a self-hosted agent to run the pipeline (`.azuredevops/ado-ci-pipeline-self-hosted.yml`). -The Microsoft hosted version is easiest to start with and recommended. Where you may consider switching to the self-hosted version, is when you have added several directories under `src` that have individual containers and the size of all the docker builds in the CI pipeline comes up against the 10GB disk storage limit for Microsoft hosted pipelines (see [resource limitations of Microsoft hosted agents](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#capabilities-and-limitations)). In this case (or when other resource constraints are met) switching to a self-hosted agent pipeline may be an option and the template at `.azuredevops/ado-ci-pipeline-self-hosted.yml` includes additional steps to help manage space consumed by CI pipeline runs. The two versions are otherwise identitical in terms of building each docker container under `src`, running pytest within each of these containers and publishing test results and coverage information. +The Microsoft hosted version is easiest to start with and recommended. Where you may consider switching to the self-hosted version, is when you have added several directories under `src` that have individual containers and the size of all the docker builds in the CI pipeline comes up against the 10GB disk storage limit for Microsoft hosted pipelines (see [resource limitations of Microsoft hosted agents](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#capabilities-and-limitations)). In this case (or when other resource constraints are met) switching to a self-hosted agent pipeline may be an option and the template at `.azuredevops/ado-ci-pipeline-self-hosted.yml` includes additional steps to help manage space consumed by CI pipeline runs. The self-hosted template supports both Linux and macOS (including Apple Silicon) agents. The two versions are otherwise identical in terms of building each docker container under `src`, running pytest within each of these containers and publishing test results and coverage information. ### How to Configure Github Actions CI Pipeline @@ -209,7 +196,7 @@ ssh-add ## Contributing -This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit . +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit . When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. diff --git a/ci-tests.sh b/ci-tests.sh deleted file mode 100755 index 796b777..0000000 --- a/ci-tests.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -: ' -This script will run all unit tests in the repository (for all directories under src/ that -have at least one test_*.py under a tests folder). It will build a Docker image for each directory with tests, -using the Dockerfile in the .devcontainer directory. It will then run pytest in the Docker container -and save the test results and coverage report to the build artifacts directory. This script can be run -locally or also in an ADO CI pipeline or Github Actions CI pipeline. See the -.azuredevops/ado-ci-pipeline-ms-hosted.yml file for an example use in an ADO CI pipeline and the -.github/workflows/ci.yaml for an example use in Github Actions pipeline. -' - -set -eE - -repo_root="$(pwd)" - -# Find all the 'src' subdirectories with a 'tests' folder, extract the dir name as test_dir_parent -for test_dir_parent in $(find "${repo_root}/src" -type d -name 'tests' -exec dirname {} \; | sed "s|${repo_root}/src/||"); do - # Check for at least one Python file in the 'tests' subdirectory of test_dir_parent - count_test_py_files=$(find "${repo_root}/src/${test_dir_parent}/tests"/*.py 2>/dev/null | wc -l) - if [ $count_test_py_files != 0 ]; then - # Use the devcontainer Dockerfile to build a Docker image for the module to run tests - docker build "${repo_root}" -f "${repo_root}/src/${test_dir_parent}/.devcontainer/Dockerfile" -t "${test_dir_parent}" - - echo "Running tests for ${test_dir_parent}, found ${count_test_py_files} test files" - - : ' - Run the tests in the built Docker container, saving the test results and coverage report to /tmp/artifact_output. - Some other key parts of the docker run command are explained here: - - The local /tmp dir is mounted to docker /tmp so that there are no permission issues with the docker user and the - pipeline user that runs this script and the user that accesses the test results and coverage report artifacts. - - The --cov-append option tells pytest coverage to append the results to the existing coverage data, instead of - overwriting it, this builds up coverage for each $test_dir_parent in a single coverage report for publishing. - - Set the .coverage location to be under /tmp so it is writable, coverage.py uses this file to store intermediate - data while measuring code coverage across multiple test runs or when combining data from multiple sources. - - exit with pytest exit code to ensure script exits with non-zero exit code if pytest fails, this ensure the CI - pipeline in ADO fails if any tests fail. - ' - docker run \ - -v "${repo_root}:/workspace" \ - -v "/tmp:/tmp" \ - --env test_dir_parent="$test_dir_parent" \ - --env COVERAGE_FILE=/tmp/artifact_output/.coverage \ - "${test_dir_parent}" \ - /bin/bash -ec ' - mkdir -p /tmp/artifact_output/$test_dir_parent; \ - env "PATH=$PATH" \ - env "PYTHONPATH=/workspace/src/$test_dir_parent:$PYTHONPATH" \ - pytest \ - --junitxml=/tmp/artifact_output/$test_dir_parent/test-results-$test_dir_parent.xml \ - -o junit_suite_name=$test_dir_parent \ - --doctest-modules \ - --cov \ - --cov-config=/workspace/pyproject.toml \ - --cov-report=xml:/tmp/artifact_output/coverage.xml \ - --cov-append \ - /workspace/src/$test_dir_parent; \ - exit $?' - fi -done - -: ' -If running CI on ADO with MS-hosted agents, copy the test and coverage results to the build artifacts directory -so that it is preserved for publishing. See the .azuredevops/ado-ci-pipeline-ms-hosted.yml file for how the -BUILD_ARTIFACTSTAGINGDIRECTORY is set. -' -if [ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then - cp -r /tmp/artifact_output/* "${BUILD_ARTIFACTSTAGINGDIRECTORY}" -fi diff --git a/notebooks/.devcontainer/devcontainer.json b/notebooks/.devcontainer/devcontainer.json index d73602e..bd54af0 100644 --- a/notebooks/.devcontainer/devcontainer.json +++ b/notebooks/.devcontainer/devcontainer.json @@ -1,25 +1,32 @@ { "name": "DSToolkit Notebooks Dev Container", "build": { - // use root directory as build context so that requirements-dev.txt is accessible during build "context": "../../", "dockerfile": "Dockerfile" }, "shutdownAction": "none", + "updateRemoteUserUID": false, "features": { "ghcr.io/devcontainers/features/common-utils:2": { "installZsh": true, "configureZshAsDefaultShell": true, "installOhMyZsh": true, "upgradePackages": false, - "username": "devuser", - }, + "username": "devuser" + } + }, + "initializeCommand": "test -f ${localWorkspaceFolder}/.env || touch ${localWorkspaceFolder}/.env", + "containerEnv": { + "PYTHONPATH": "/workspaces/repo" }, "runArgs": [ "--env-file", - "../.env" + "${localWorkspaceFolder}/.env" ], - "postCreateCommand": "pre-commit install --overwrite", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/repo,type=bind", + "workspaceFolder": "/workspaces/repo", + "remoteUser": "devuser", + "postCreateCommand": "cd /workspaces/repo && pre-commit install --overwrite || echo 'pre-commit installation failed; continuing without pre-commit hooks'", "customizations": { "vscode": { "extensions": [ @@ -66,7 +73,7 @@ "notebook.source.fixAll": "explicit", "notebook.source.organizeImports": "explicit" } - }, + } } - }, + } } diff --git a/pyproject.toml b/pyproject.toml index e3bfcf0..75e5459 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ extend-exclude = ["notebooks"] # - pep8-naming (N) select = ["E", "F", "B", "S", "I", "N"] - [tool.ruff.lint.per-file-ignores] "**/tests/**/test_*.py" = [ "S101", # asserts allowed in tests @@ -24,9 +23,21 @@ select = ["E", "F", "B", "S", "I", "N"] pythonpath = "src" [tool.coverage.run] +source = ["src"] +relative_files = true omit = [ # ignore all notebooks in src "*/notebooks/*", # ignore all tests in src "*/tests/*", ] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "if __name__ == .__main__.:", +] + +[tool.coverage.xml] +output = "coverage.xml" diff --git a/src/sample_cpu_project/.devcontainer/devcontainer.json b/src/sample_cpu_project/.devcontainer/devcontainer.json index 839d3dc..ecfb136 100644 --- a/src/sample_cpu_project/.devcontainer/devcontainer.json +++ b/src/sample_cpu_project/.devcontainer/devcontainer.json @@ -1,25 +1,32 @@ { "name": "Sample CPU Project Dev Container", "build": { - // use root directory as build context so that requirements-dev.txt is accessible during build "context": "../../../", "dockerfile": "Dockerfile" }, "shutdownAction": "none", + "updateRemoteUserUID": false, "features": { "ghcr.io/devcontainers/features/common-utils:2": { "installZsh": true, "configureZshAsDefaultShell": true, "installOhMyZsh": true, "upgradePackages": false, - "username": "devuser", - }, + "username": "devuser" + } + }, + "initializeCommand": "test -f ${localWorkspaceFolder}/.env || touch ${localWorkspaceFolder}/.env", + "containerEnv": { + "PYTHONPATH": "/workspaces/repo" }, "runArgs": [ "--env-file", - "../../.env" + "${localWorkspaceFolder}/.env" ], - "postCreateCommand": "pre-commit install --overwrite", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/repo,type=bind", + "workspaceFolder": "/workspaces/repo", + "remoteUser": "devuser", + "postCreateCommand": "cd /workspaces/repo && pre-commit install --overwrite || echo 'pre-commit installation failed; continuing without pre-commit hooks'", "customizations": { "vscode": { "extensions": [ @@ -65,8 +72,8 @@ "notebook.codeActionsOnSave": { "notebook.source.fixAll": "explicit", "notebook.source.organizeImports": "explicit" - }, - }, + } + } } - }, + } } diff --git a/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json b/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json index 725a88f..bc16b5b 100644 --- a/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json +++ b/src/sample_pytorch_gpu_project/.devcontainer/devcontainer.json @@ -1,27 +1,35 @@ { "name": "Sample PyTorch GPU Project Dev Container", "build": { - // use root directory as build context so that requirements-dev.txt is accessible during build "context": "../../../", "dockerfile": "Dockerfile" }, "shutdownAction": "none", + "updateRemoteUserUID": false, "features": { "ghcr.io/devcontainers/features/common-utils:2": { "installZsh": true, "configureZshAsDefaultShell": true, "installOhMyZsh": true, "upgradePackages": false, - "username": "devuser", - }, + "username": "devuser" + } + }, + "initializeCommand": "test -f ${localWorkspaceFolder}/.env || touch ${localWorkspaceFolder}/.env", + "containerEnv": { + "PYTHONPATH": "/workspaces/repo" }, "runArgs": [ - "--gpus", - "all", "--env-file", - "../../.env" + "${localWorkspaceFolder}/.env" ], - "postCreateCommand": "pre-commit install --overwrite", + "hostRequirements": { + "gpu": "optional" + }, + "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/repo,type=bind", + "workspaceFolder": "/workspaces/repo", + "remoteUser": "devuser", + "postCreateCommand": "cd /workspaces/repo && pre-commit install --overwrite || echo 'pre-commit installation failed; continuing without pre-commit hooks'", "customizations": { "vscode": { "extensions": [ @@ -68,7 +76,7 @@ "notebook.source.fixAll": "explicit", "notebook.source.organizeImports": "explicit" } - }, + } } - }, + } }