to prefect flow, uv, ruff, ty

.env.template

@@ -1,16 +1,27 @@
+LOG_LEVEL=INFO
+
+INFLUXDB_HOSTNAME=omotes_influxdb
+INFLUXDB_PORT=8096
 INFLUXDB_USERNAME=root
 INFLUXDB_PASSWORD=9012
-INFLUXDB_HOSTNAME=omotes_influxdb
-INFLUXDB_PORT=8086
-INFLUXDB_RPC_PORT=8088
 
-RABBITMQ_USERNAME=celery
-RABBITMQ_PASSWORD=somepass2
-RABBITMQ_HOSTNAME=rabbitmq-nwn
-RABBITMQ_PORT=5672
-RABBITMQ_VIRTUALHOST=omotes_celery
+MINIO_HOST=omotes-minio
+MINIO_PORT=9000
+MINIO_EXTERNAL_URL=omotes-minio:9000
+MINIO_ACCESS_KEY=omotesadmin
+MINIO_SECRET=password
 
-LOG_LEVEL=INFO
+PREFECT_API_URL=http://localhost:4200/api
+PREFECT_API_URL_FOR_WORKER=http://omotes-prefect-server:4200/api
+PREFECT_WORK_POOL_NAME="docker-worker"
+PREFECT_API_AUTH_STRING=admin:password
+
+PREFECT_FLOW_MAX_CONCURRENT_RUNS=4
+PREFECT_FLOW_TIMEOUT_SECONDS=172800
 
-TASK_RESULT_QUEUE_NAME=omotes_task_result_events
-TASK_PROGRESS_QUEUE_NAME=omotes_task_progress_events
+# For deploying to prefect (will run inside local docker container or k8s pod)
+# When using docker desktop you can use your local code without pushing to github first:
+PREFECT_USE_LOCAL_CODE_AND_IMAGE=true
+PREFECT_USE_LOCAL_SDK=true
+# SIMULATOR_WORKER_VERSION not used when PREFECT_USE_LOCAL_CODE_AND_IMAGE=true:
+SIMULATOR_WORKER_VERSION=0.0.0

.env.template.local

@@ -1,16 +1,27 @@
+LOG_LEVEL=INFO
+
+INFLUXDB_HOSTNAME=omotes_influxdb
+INFLUXDB_PORT=8096
 INFLUXDB_USERNAME=root
 INFLUXDB_PASSWORD=9012
-INFLUXDB_HOSTNAME=localhost
-INFLUXDB_PORT=8096
-INFLUXDB_RPC_PORT=8088
 
-RABBITMQ_USERNAME=celery
-RABBITMQ_PASSWORD=somepass2
-RABBITMQ_HOSTNAME=localhost
-RABBITMQ_PORT=5672
-RABBITMQ_VIRTUALHOST=omotes_celery
+MINIO_HOST=omotes-minio
+MINIO_PORT=9000
+MINIO_EXTERNAL_URL=omotes-minio:9000
+MINIO_ACCESS_KEY=omotesadmin
+MINIO_SECRET=password
 
-LOG_LEVEL=INFO
+PREFECT_API_URL=http://localhost:4200/api
+PREFECT_API_URL_FOR_WORKER=http://omotes-prefect-server:4200/api
+PREFECT_WORK_POOL_NAME="docker-worker"
+PREFECT_API_AUTH_STRING=admin:password
+
+PREFECT_FLOW_MAX_CONCURRENT_RUNS=4
+PREFECT_FLOW_TIMEOUT_SECONDS=172800
 
-TASK_RESULT_QUEUE_NAME=omotes_task_result_events
-TASK_PROGRESS_QUEUE_NAME=omotes_task_progress_events
+# For deploying to prefect (will run inside local docker container or k8s pod)
+# When using docker desktop you can use your local code without pushing to github first:
+PREFECT_USE_LOCAL_CODE_AND_IMAGE=true
+PREFECT_USE_LOCAL_SDK=true
+# SIMULATOR_WORKER_VERSION not used when PREFECT_USE_LOCAL_CODE_AND_IMAGE=true:
+SIMULATOR_WORKER_VERSION=0.0.0

.github/workflows/ci.yml

@@ -1,123 +1,86 @@
-name: Build-Test-Lint-etc (linux)
+name: Build-Test-Lint-Security-Typecheck
 
 on: [push]
 
 jobs:
-  setup:
+  lint:
+    name: Lint
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [ "3.11" ]
-    name: Setup
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+      - uses: actions/checkout@v4
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-      - run: |
-          ./ci/linux/create_venv.sh
-          ./ci/linux/install_dependencies.sh
-      - name: 'Tar venv'
-        run: tar -cvf venv.tar ./.venv/
-      - name: Save venv
-        uses: actions/upload-artifact@v4
-        with:
-          path: ./venv.tar
-          name: ${{ runner.os }}-python-${{ matrix.python-version }}-venv-${{github.run_id}}
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run linter
+        run: just lint
+      - name: Check formatting
+        run: just format-check
 
-  lint:
-    name: Lint
+  security:
+    name: Security Check
     runs-on: ubuntu-latest
-    needs: [ setup ]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [ "3.11" ]
     steps:
-    - uses: actions/checkout@v3
-    - name: Restore venv
-      uses: actions/download-artifact@v4
-      with:
-        name: ${{ runner.os }}-python-${{ matrix.python-version }}-venv-${{github.run_id}}
-    - name: Untar venv
-      run: tar -xvf venv.tar
-    - name: run unit tests
-      run: |
-        ./ci/linux/lint.sh
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run security checks
+        run: just security
 
   test:
     name: Test
     runs-on: ubuntu-latest
-    needs: [ setup ]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [ "3.11" ]
     steps:
-    - uses: actions/checkout@v3
-    - name: Restore venv
-      uses: actions/download-artifact@v4
-      with:
-        name: ${{ runner.os }}-python-${{ matrix.python-version }}-venv-${{github.run_id}}
-    - name: Untar venv
-      run: tar -xvf venv.tar
-    - name: run unit tests
-      run: |
-        ./ci/linux/test_unit.sh
-
-    - name: Upload debug ESDL artifacts
-      if: always()
-      uses: actions/upload-artifact@v4
-      with:
-        name: debug-esdl
-        path: debug_esdl_*
-        if-no-files-found: warn
-
-    - name: Surface failing tests
-      if: always()
-      uses: pmeier/pytest-results-action@main
-      with:
-        # A list of JUnit XML files, directories containing the former, and wildcard
-        # patterns to process.
-        # See @actions/glob for supported patterns.
-        path: test-results.xml
-
-        # Add a summary of the results at the top of the report
-        # Default: true
-        summary: true
-
-        # Select which results should be included in the report.
-        # Follows the same syntax as
-        # `pytest -r`
-        # Default: fEX
-        display-options: fEX
-
-        # Fail the workflow if no JUnit XML was found.
-        # Default: true
-        fail-on-empty: true
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run unit tests
+        run: just test
+      - name: Publish test results
+        if: always()
+        uses: pmeier/pytest-results-action@main
+        with:
+          path: test-results.xml
+          summary: true
+          display-options: fEX
+          fail-on-empty: true
 
   typecheck:
-    name: Typecheck
+    name: Type Check
     runs-on: ubuntu-latest
-    needs: [ setup ]
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: [ "3.11" ]
     steps:
-    - uses: actions/checkout@v3
-    - name: Restore venv
-      uses: actions/download-artifact@v4
-      with:
-        name: ${{ runner.os }}-python-${{ matrix.python-version }}-venv-${{github.run_id}}
-    - name: Untar venv
-      run: tar -xvf venv.tar
-    - name: Add mypy annotator
-      uses: pr-annotators/mypy-pr-annotator@v1.0.0
-
-    - name: run typechecker
-      run: |
-        ./ci/linux/typecheck.sh
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+      - name: Install just
+        uses: taiki-e/install-action@just
+      - name: Install dependencies
+        run: just install
+      - name: Run type checker
+        run: just typecheck

.github/workflows/publish_container_image.yml

@@ -1,28 +1,48 @@
-name: Deply to GitHub Container Registry
+name: Publish Container Image
+run-name: Releasing next version 🚀
+
 on:
   push:
     tags:
-      - '*'
+      - "*"
 
 jobs:
- publish-container-image:
-   runs-on: ubuntu-latest
-   steps:
-   - name: Checkout sources
-     uses: actions/checkout@v3
-     with:
-      fetch-depth: 0       
+  publish-container-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-   - name: Login to GitHub Container Registry
-     uses: docker/login-action@v1
-     with:
-       registry: ghcr.io
-       username: ${{ github.actor }}
-       password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/project-omotes/omotes-simulator-worker
+          tags: |
+            type=raw,value=latest
+            type=raw,value=${{ github.ref_name }}
 
-   - name: Build the container image
-     run: |
-      docker build . --tag ghcr.io/project-omotes/omotes-simulator-worker:latest --tag ghcr.io/project-omotes/omotes-simulator-worker:${{github.ref_name}}
-      docker push ghcr.io/project-omotes/omotes-simulator-worker:latest
-      docker push ghcr.io/project-omotes/omotes-simulator-worker:${{github.ref_name}}
+      - name: Build and push container image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 

.gitignore

@@ -218,7 +218,7 @@ fabric.properties
 !.idea/codeStyles
 !.idea/runConfigurations
 
-unit_test_coverage/
+tests_coverage/
 test-results.xml
 debug_esdl_*/
 

.vscode/launch.json

@@ -1,15 +1,44 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Python Debugger: Current File",
+            "name": "prefect_deploy_flow",
             "type": "debugpy",
             "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal"
+            "program": "${workspaceFolder}/src/simulator_worker/prefect_deploy_flow.py",
+            "cwd": "${workspaceFolder}/src",
+            "console": "integratedTerminal",
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/src"
+            },
+            "envFile": "${workspaceFolder}/.env"
+        },
+        {
+            "name": "run_simulator_flow_function",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/local_run/run_simulator_flow_function.py",
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal",
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/src"
+            },
+            "envFile": "${workspaceFolder}/.env"
+        },
+        {
+            "name": "pytest",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "pytest",
+            "args": [
+                "tests",
+                "-q"
+            ],
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/src"
+            },
+            "console": "integratedTerminal",
+            "justMyCode": false
         }
     ]
 }