diff --git a/.github/workflows/call-docker-build-result-dev.yaml b/.github/workflows/call-docker-build-result-dev.yaml new file mode 100644 index 0000000000..dc28e5d679 --- /dev/null +++ b/.github/workflows/call-docker-build-result-dev.yaml @@ -0,0 +1,507 @@ + +name: Build Result Service + +on: + # workflow_dispatch: + push: + branches: + - development + paths: + - 'result/**' + - '.github/workflows/call-docker-build-result-dev.yaml' + # # pull_request: + # branches: + # - development + # paths: + # - 'result/**' + # - '.github/workflows/call-docker-build-result-dev.yaml' + +jobs: + build-scan-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Prepare security report directory + run: | + mkdir -p security-reports + chmod 777 security-reports + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-southeast-1 + + - name: Run SAST scan with Semgrep + run: | + python -m pip install --upgrade pip + python -m pip install semgrep + semgrep scan \ + --config p/ci \ + --severity ERROR \ + --exclude result/views/angular.min.js \ + --exclude result/views/socket.io.js \ + --sarif \ + --output security-reports/semgrep-result.sarif \ + . || true + + - name: Generate SAST HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/semgrep-result.sarif") + html_path = pathlib.Path("security-reports/semgrep-result.html") + title = "SAST (Semgrep) - Result Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish SAST Summary + if: always() + run: | + { + echo "## SAST (Semgrep) - Result Service" + echo "" + if [ -f security-reports/semgrep-result.sarif ]; then + echo "✅ SAST scan completed." + echo "" + echo "**Report artifact files:** semgrep-result.sarif, semgrep-result.html" + else + echo "❌ SAST report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run dependency vulnerability scan with Trivy + uses: aquasecurity/trivy-action@v0.20.0 + with: + scan-type: fs + scan-ref: ./result + vuln-type: library + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: '0' + format: sarif + output: security-reports/trivy-fs-result.sarif + + - name: Generate Dependency Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/trivy-fs-result.sarif") + html_path = pathlib.Path("security-reports/trivy-fs-result.html") + title = "SCA / Dependency Scan (Trivy FS) - Result Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Dependency Scan Summary + if: always() + run: | + { + echo "## SCA / Dependency Vulnerability Scan (Trivy FS) - Result Service" + echo "" + if [ -f security-reports/trivy-fs-result.sarif ]; then + echo "✅ Dependency vulnerability scan completed." + echo "" + echo "**Report artifact files:** trivy-fs-result.sarif, trivy-fs-result.html" + else + echo "❌ Dependency scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run secret scan with Gitleaks + run: | + GITLEAKS_VERSION=8.24.2 + curl -sSL -o gitleaks.tar.gz "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + tar -xzf gitleaks.tar.gz gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks detect \ + --source . \ + --redact \ + --verbose \ + --report-format json \ + --report-path security-reports/gitleaks-result.json \ + --exit-code 0 + + - name: Generate Secret Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + json_path = pathlib.Path("security-reports/gitleaks-result.json") + html_path = pathlib.Path("security-reports/gitleaks-result.html") + title = "Secret Scan (Gitleaks) - Result Service" + findings = [] + error = None + try: + text = json_path.read_text(encoding="utf-8").strip() + data = json.loads(text) if text else [] + if isinstance(data, list): + findings = data + elif isinstance(data, dict): + findings = data.get("findings") or data.get("results") or [] + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source JSON file: {html.escape(str(json_path))}

" + ] + if error: + parts.append(f"

Could not parse Gitleaks report: {html.escape(error)}

") + elif not findings: + parts.append("

No secrets were reported.

") + else: + parts.append(f"

Total findings: {len(findings)}

") + parts.append("") + for item in findings: + rule = item.get("RuleID") or item.get("rule") or "" + desc = item.get("Description") or item.get("description") or "" + file = item.get("File") or item.get("file") or "" + line = item.get("StartLine") or item.get("line") or "" + commit = item.get("Commit") or item.get("commit") or "" + parts.append("" + "".join(f"" for x in (rule, desc, file, line, commit)) + "") + parts.append("
RuleDescriptionFileLineCommit
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Secret Scan Summary + if: always() + run: | + { + echo "## Secret Scan (Gitleaks) - Result Service" + echo "" + if [ -f security-reports/gitleaks-result.json ]; then + echo "✅ Secret scan completed." + echo "" + echo "**Report artifact files:** gitleaks-result.json, gitleaks-result.html" + else + echo "❌ Secret scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Set version tag + run: | + if [[ "${GITHUB_REF##*/}" == "main" ]]; then + echo "TAG=v2" >> $GITHUB_ENV + else + echo "TAG=v1" >> $GITHUB_ENV + fi + + - name: Build Docker image + run: | + REGISTRY=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev + docker build -t result-service:${TAG} -t $REGISTRY:result-${TAG} ./result + + - name: Security scan with Trivy + uses: aquasecurity/trivy-action@v0.20.0 + with: + scan-type: image + image-ref: result-service:${{ env.TAG }} + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: '0' + format: sarif + output: security-reports/trivy-image-result.sarif + + - name: Generate Container Image Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/trivy-image-result.sarif") + html_path = pathlib.Path("security-reports/trivy-image-result.html") + title = "Container Image Scan (Trivy Image) - Result Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Container Image Scan Summary + if: always() + run: | + { + echo "## Container Image Scan (Trivy Image) - Result Service" + echo "" + if [ -f security-reports/trivy-image-result.sarif ]; then + echo "✅ Container image scan completed." + echo "" + echo "**Report artifact files:** trivy-image-result.sarif, trivy-image-result.html" + else + echo "❌ Container image scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run DAST scan with OWASP ZAP + continue-on-error: true + run: | + docker network create result-dast-net || true + docker rm -f db-service result-dast || true + docker run -d --name db-service --network result-dast-net \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=postgres \ + postgres:15-alpine + docker run -d --name result-dast --network result-dast-net -p 8081:80 result-service:${TAG} + sleep 30 + curl --fail --retry 10 --retry-delay 5 http://localhost:8081/ + + mkdir -p zap-reports + chmod 777 zap-reports + + docker run --rm \ + --user root \ + --network host \ + -v "${PWD}/zap-reports:/zap/wrk/:rw" \ + ghcr.io/zaproxy/zaproxy:stable \ + zap-baseline.py \ + -t http://localhost:8081 \ + -I \ + -r result-zap-report.html + + mkdir -p security-reports + cp zap-reports/result-zap-report.html security-reports/result-zap-report.html || true + echo "=== zap-reports ===" + ls -lah zap-reports || true + echo "=== zap-reports ===" + ls -lah zap-reports || true + echo "=== security-reports ===" + ls -lah security-reports || true || true + + - name: Verify ZAP Report + if: always() + run: | + echo "=== zap-reports ===" + ls -lah zap-reports || true + echo "=== security-reports ===" + ls -lah security-reports || true + + - name: Publish DAST Summary + if: always() + run: | + { + echo "## DAST (OWASP ZAP) - Result Service" + echo "" + if [ -f security-reports/result-zap-report.html ]; then + echo "✅ DAST scan completed and HTML report was generated." + echo "" + echo "**Report artifact file:** result-zap-report.html" + else + echo "❌ DAST report was not generated." + echo "" + echo "Please check the OWASP ZAP logs above." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Publish Consolidated Security Summary + if: always() + run: | + { + echo "# DevSecOps Security Reports - Result Service" + echo "" + echo "| Security Control | Report File |" + echo "|---|---|" + echo "| SAST (Semgrep) | semgrep-result.sarif, semgrep-result.html |" + echo "| SCA / Dependency Scan (Trivy FS) | trivy-fs-result.sarif, trivy-fs-result.html |" + echo "| Secret Scan (Gitleaks) | gitleaks-result.json, gitleaks-result.html |" + echo "| Container Image Scan (Trivy Image) | trivy-image-result.sarif, trivy-image-result.html |" + echo "| DAST (OWASP ZAP) | result-zap-report.html |" + echo "" + echo "Human-readable HTML reports and machine-readable SARIF/JSON reports are uploaded as the **security-reports-result** workflow artifact." + echo "" + echo "## Files generated" + echo "" + ls -lah security-reports || true + echo "" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Security Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-reports-result + path: security-reports/ + if-no-files-found: warn + + - name: Push image to ECR + run: | + REGISTRY=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev + docker push $REGISTRY:result-${TAG} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --region ap-southeast-1 \ + --name bq-eks-cluster-dev + + - name: Deploy application + run: | + kubectl apply -f k8s-specifications/ + + - name: Update deployment + run: | + kubectl set image deployment/result \ + result=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev:result-${TAG} + + - name: Wait for rollout + run: | + kubectl rollout status deployment/result + + diff --git a/.github/workflows/call-docker-build-result.yaml b/.github/workflows/call-docker-build-result.yaml deleted file mode 100644 index a946a87b03..0000000000 --- a/.github/workflows/call-docker-build-result.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: Build Result -# template source: https://github.com/dockersamples/.github/blob/main/templates/call-docker-build.yaml - -on: - # we want pull requests so we can build(test) but not push to image registry - push: - branches: - - 'main' - # only build when important files change - paths: - - 'result/**' - - '.github/workflows/call-docker-build-result.yaml' - pull_request: - branches: - - 'main' - # only build when important files change - paths: - - 'result/**' - - '.github/workflows/call-docker-build-result.yaml' - -jobs: - call-docker-build: - - name: Result Call Docker Build - - uses: dockersamples/.github/.github/workflows/reusable-docker-build.yaml@main - - permissions: - contents: read - packages: write # needed to push docker image to ghcr.io - pull-requests: write # needed to create and update comments in PRs - - secrets: - - # Only needed if with:dockerhub-enable is true below - dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} - - # Only needed if with:dockerhub-enable is true below - dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} - - with: - - ### REQUIRED - ### ENABLE ONE OR BOTH REGISTRIES - ### tell docker where to push. - ### NOTE if Docker Hub is set to true, you must set secrets above and also add account/repo/tags below - dockerhub-enable: true - ghcr-enable: true - - ### REQUIRED - ### A list of the account/repo names for docker build. List should match what's enabled above - ### defaults to: - image-names: | - ghcr.io/dockersamples/example-voting-app-result - dockersamples/examplevotingapp_result - - ### REQUIRED set rules for tagging images, based on special action syntax: - ### https://github.com/docker/metadata-action#tags-input - ### defaults to: - tag-rules: | - type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=raw,value=before,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=raw,value=after,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=ref,event=pr - - ### path to where docker should copy files into image - ### defaults to root of repository (.) - context: result - - ### Dockerfile alternate name. Default is Dockerfile (relative to context path) - # file: Containerfile - - ### build stage to target, defaults to empty, which builds to last stage in Dockerfile - # target: - - ### platforms to build for, defaults to linux/amd64 - ### other options: linux/amd64,linux/arm64,linux/arm/v7 - platforms: linux/amd64,linux/arm64,linux/arm/v7 - - ### Create a PR comment with image tags and labels - ### defaults to false - # comment-enable: false diff --git a/.github/workflows/call-docker-build-vote-dev.yaml b/.github/workflows/call-docker-build-vote-dev.yaml new file mode 100644 index 0000000000..054c89a5b6 --- /dev/null +++ b/.github/workflows/call-docker-build-vote-dev.yaml @@ -0,0 +1,502 @@ +name: Build Vote Service + +on: + # workflow_dispatch: + push: + branches: + - development + paths: + - 'vote/**' + - '.github/workflows/call-docker-build-vote-dev.yaml' + # pull_request: + # branches: + # - main + # - development + # paths: + # - 'vote/**' + # - '.github/workflows/call-docker-build-vote.yaml' + +jobs: + build-scan-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Prepare security report directory + run: | + mkdir -p security-reports + chmod 777 security-reports + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-southeast-1 + + - name: Run SAST scan with Semgrep + run: | + python -m pip install --upgrade pip + python -m pip install semgrep + semgrep scan \ + --config p/ci \ + --severity ERROR \ + --exclude result/views/angular.min.js \ + --exclude result/views/socket.io.js \ + --sarif \ + --output security-reports/semgrep-vote.sarif \ + . || true + + - name: Generate SAST HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/semgrep-vote.sarif") + html_path = pathlib.Path("security-reports/semgrep-vote.html") + title = "SAST (Semgrep) - Vote Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish SAST Summary + if: always() + run: | + { + echo "## SAST (Semgrep) - Vote Service" + echo "" + if [ -f security-reports/semgrep-vote.sarif ]; then + echo "✅ SAST scan completed." + echo "" + echo "**Report artifact files:** semgrep-vote.sarif, semgrep-vote.html" + else + echo "❌ SAST report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run dependency vulnerability scan with Trivy + uses: aquasecurity/trivy-action@v0.20.0 + with: + scan-type: fs + scan-ref: ./vote + vuln-type: library + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: '0' + format: sarif + output: security-reports/trivy-fs-vote.sarif + + - name: Generate Dependency Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/trivy-fs-vote.sarif") + html_path = pathlib.Path("security-reports/trivy-fs-vote.html") + title = "SCA / Dependency Scan (Trivy FS) - Vote Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Dependency Scan Summary + if: always() + run: | + { + echo "## SCA / Dependency Vulnerability Scan (Trivy FS) - Vote Service" + echo "" + if [ -f security-reports/trivy-fs-vote.sarif ]; then + echo "✅ Dependency vulnerability scan completed." + echo "" + echo "**Report artifact files:** trivy-fs-vote.sarif, trivy-fs-vote.html" + else + echo "❌ Dependency scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run secret scan with Gitleaks + run: | + GITLEAKS_VERSION=8.24.2 + curl -sSL -o gitleaks.tar.gz "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + tar -xzf gitleaks.tar.gz gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks detect \ + --source . \ + --redact \ + --verbose \ + --report-format json \ + --report-path security-reports/gitleaks-vote.json \ + --exit-code 0 + + - name: Generate Secret Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + json_path = pathlib.Path("security-reports/gitleaks-vote.json") + html_path = pathlib.Path("security-reports/gitleaks-vote.html") + title = "Secret Scan (Gitleaks) - Vote Service" + findings = [] + error = None + try: + text = json_path.read_text(encoding="utf-8").strip() + data = json.loads(text) if text else [] + if isinstance(data, list): + findings = data + elif isinstance(data, dict): + findings = data.get("findings") or data.get("results") or [] + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source JSON file: {html.escape(str(json_path))}

" + ] + if error: + parts.append(f"

Could not parse Gitleaks report: {html.escape(error)}

") + elif not findings: + parts.append("

No secrets were reported.

") + else: + parts.append(f"

Total findings: {len(findings)}

") + parts.append("") + for item in findings: + rule = item.get("RuleID") or item.get("rule") or "" + desc = item.get("Description") or item.get("description") or "" + file = item.get("File") or item.get("file") or "" + line = item.get("StartLine") or item.get("line") or "" + commit = item.get("Commit") or item.get("commit") or "" + parts.append("" + "".join(f"" for x in (rule, desc, file, line, commit)) + "") + parts.append("
RuleDescriptionFileLineCommit
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Secret Scan Summary + if: always() + run: | + { + echo "## Secret Scan (Gitleaks) - Vote Service" + echo "" + if [ -f security-reports/gitleaks-vote.json ]; then + echo "✅ Secret scan completed." + echo "" + echo "**Report artifact files:** gitleaks-vote.json, gitleaks-vote.html" + else + echo "❌ Secret scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Set version tag + run: | + if [[ "${GITHUB_REF##*/}" == "main" ]]; then + echo "TAG=v2" >> $GITHUB_ENV + else + echo "TAG=v1" >> $GITHUB_ENV + fi + + - name: Build Docker image + run: | + REGISTRY=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev + docker build -t vote-service:${TAG} ./vote + docker tag vote-service:${TAG} $REGISTRY:vote-${TAG} + + - name: Security scan with Trivy + uses: aquasecurity/trivy-action@v0.20.0 + with: + scan-type: image + image-ref: vote-service:${{ env.TAG }} + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: '0' + format: sarif + output: security-reports/trivy-image-vote.sarif + + - name: Generate Container Image Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/trivy-image-vote.sarif") + html_path = pathlib.Path("security-reports/trivy-image-vote.html") + title = "Container Image Scan (Trivy Image) - Vote Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Container Image Scan Summary + if: always() + run: | + { + echo "## Container Image Scan (Trivy Image) - Vote Service" + echo "" + if [ -f security-reports/trivy-image-vote.sarif ]; then + echo "✅ Container image scan completed." + echo "" + echo "**Report artifact files:** trivy-image-vote.sarif, trivy-image-vote.html" + else + echo "❌ Container image scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run DAST scan with OWASP ZAP + continue-on-error: true + run: | + docker network create vote-dast-net || true + docker rm -f redis vote-dast || true + docker run -d --name redis --network vote-dast-net redis:7-alpine + docker run -d --name vote-dast --network vote-dast-net -p 8080:80 vote-service:${TAG} + sleep 20 + curl --fail --retry 10 --retry-delay 5 http://localhost:8080/ + + mkdir -p zap-reports + chmod 777 zap-reports + + docker run --rm \ + --user root \ + --network host \ + -v "${PWD}/zap-reports:/zap/wrk/:rw" \ + ghcr.io/zaproxy/zaproxy:stable \ + zap-baseline.py \ + -t http://localhost:8080 \ + -I \ + -r vote-zap-report.html + + mkdir -p security-reports + cp zap-reports/vote-zap-report.html security-reports/vote-zap-report.html || true + echo "=== zap-reports ===" + ls -lah zap-reports || true + echo "=== zap-reports ===" + ls -lah zap-reports || true + echo "=== security-reports ===" + ls -lah security-reports || true || true + + - name: Verify ZAP Report + if: always() + run: | + echo "=== zap-reports ===" + ls -lah zap-reports || true + echo "=== security-reports ===" + ls -lah security-reports || true + + - name: Publish DAST Summary + if: always() + run: | + { + echo "## DAST (OWASP ZAP) - Vote Service" + echo "" + if [ -f security-reports/vote-zap-report.html ]; then + echo "✅ DAST scan completed and HTML report was generated." + echo "" + echo "**Report artifact file:** vote-zap-report.html" + else + echo "❌ DAST report was not generated." + echo "" + echo "Please check the OWASP ZAP logs above." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Publish Consolidated Security Summary + if: always() + run: | + { + echo "# DevSecOps Security Reports - Vote Service" + echo "" + echo "| Security Control | Report File |" + echo "|---|---|" + echo "| SAST (Semgrep) | semgrep-vote.sarif, semgrep-vote.html |" + echo "| SCA / Dependency Scan (Trivy FS) | trivy-fs-vote.sarif, trivy-fs-vote.html |" + echo "| Secret Scan (Gitleaks) | gitleaks-vote.json, gitleaks-vote.html |" + echo "| Container Image Scan (Trivy Image) | trivy-image-vote.sarif, trivy-image-vote.html |" + echo "| DAST (OWASP ZAP) | vote-zap-report.html |" + echo "" + echo "Human-readable HTML reports and machine-readable SARIF/JSON reports are uploaded as the **security-reports-vote** workflow artifact." + echo "" + echo "## Files generated" + echo "" + ls -lah security-reports || true + echo "" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Security Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-reports-vote + path: security-reports/ + if-no-files-found: warn + + - name: Push image to ECR + run: | + REGISTRY=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev + docker push $REGISTRY:vote-${TAG} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --region ap-southeast-1 \ + --name bq-eks-cluster-dev + + - name: Deploy application + run: | + kubectl apply -f k8s-specifications/ + + - name: Update deployment + run: | + kubectl set image deployment/vote \ + vote=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev:vote-${TAG} + + - name: Wait for rollout + run: | + kubectl rollout status deployment/vote \ No newline at end of file diff --git a/.github/workflows/call-docker-build-vote.yaml b/.github/workflows/call-docker-build-vote.yaml deleted file mode 100644 index cb4a484a2a..0000000000 --- a/.github/workflows/call-docker-build-vote.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: Build Vote -# template source: https://github.com/dockersamples/.github/blob/main/templates/call-docker-build.yaml - -on: - # we want pull requests so we can build(test) but not push to image registry - push: - branches: - - 'main' - # only build when important files change - paths: - - 'vote/**' - - '.github/workflows/call-docker-build-vote.yaml' - pull_request: - branches: - - 'main' - # only build when important files change - paths: - - 'vote/**' - - '.github/workflows/call-docker-build-vote.yaml' - -jobs: - call-docker-build: - - name: Vote Call Docker Build - - uses: dockersamples/.github/.github/workflows/reusable-docker-build.yaml@main - - permissions: - contents: read - packages: write # needed to push docker image to ghcr.io - pull-requests: write # needed to create and update comments in PRs - - secrets: - - # Only needed if with:dockerhub-enable is true below - dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} - - # Only needed if with:dockerhub-enable is true below - dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} - - with: - - ### REQUIRED - ### ENABLE ONE OR BOTH REGISTRIES - ### tell docker where to push. - ### NOTE if Docker Hub is set to true, you must set secrets above and also add account/repo/tags below - dockerhub-enable: true - ghcr-enable: true - - ### REQUIRED - ### A list of the account/repo names for docker build. List should match what's enabled above - ### defaults to: - image-names: | - ghcr.io/dockersamples/example-voting-app-vote - dockersamples/examplevotingapp_vote - - ### REQUIRED set rules for tagging images, based on special action syntax: - ### https://github.com/docker/metadata-action#tags-input - ### defaults to: - tag-rules: | - type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=raw,value=before,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=raw,value=after,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=ref,event=pr - - ### path to where docker should copy files into image - ### defaults to root of repository (.) - context: vote - - ### Dockerfile alternate name. Default is Dockerfile (relative to context path) - # file: Containerfile - - ### build stage to target, defaults to empty, which builds to last stage in Dockerfile - # target: - - ### platforms to build for, defaults to linux/amd64 - ### other options: linux/amd64,linux/arm64,linux/arm/v7 - platforms: linux/amd64,linux/arm64,linux/arm/v7 - - ### Create a PR comment with image tags and labels - ### defaults to false - # comment-enable: false diff --git a/.github/workflows/call-docker-build-worker-dev.yaml b/.github/workflows/call-docker-build-worker-dev.yaml new file mode 100644 index 0000000000..e0a66b0fd7 --- /dev/null +++ b/.github/workflows/call-docker-build-worker-dev.yaml @@ -0,0 +1,461 @@ +name: Build Worker Service + +on: + # workflow_dispatch: + push: + branches: + - development + paths: + - 'worker/**' + - '.github/workflows/call-docker-build-worker-dev.yaml' + # pull_request: + # branches: + # - main + # - development + # paths: + # - 'worker/**' + # - '.github/workflows/call-docker-build-worker.yaml' + +jobs: + build-scan-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Prepare security report directory + run: | + mkdir -p security-reports + chmod 777 security-reports + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-southeast-1 + + - name: Run SAST scan with Semgrep + run: | + python -m pip install --upgrade pip + python -m pip install semgrep + semgrep scan \ + --config p/ci \ + --severity ERROR \ + --exclude result/views/angular.min.js \ + --exclude result/views/socket.io.js \ + --sarif \ + --output security-reports/semgrep-worker.sarif \ + . || true + + - name: Generate SAST HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/semgrep-worker.sarif") + html_path = pathlib.Path("security-reports/semgrep-worker.html") + title = "SAST (Semgrep) - Worker Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish SAST Summary + if: always() + run: | + { + echo "## SAST (Semgrep) - Worker Service" + echo "" + if [ -f security-reports/semgrep-worker.sarif ]; then + echo "✅ SAST scan completed." + echo "" + echo "**Report artifact files:** semgrep-worker.sarif, semgrep-worker.html" + else + echo "❌ SAST report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run dependency vulnerability scan with Trivy + uses: aquasecurity/trivy-action@v0.20.0 + with: + scan-type: fs + scan-ref: ./worker + vuln-type: library + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: '0' + format: sarif + output: security-reports/trivy-fs-worker.sarif + + - name: Generate Dependency Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/trivy-fs-worker.sarif") + html_path = pathlib.Path("security-reports/trivy-fs-worker.html") + title = "SCA / Dependency Scan (Trivy FS) - Worker Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Dependency Scan Summary + if: always() + run: | + { + echo "## SCA / Dependency Vulnerability Scan (Trivy FS) - Worker Service" + echo "" + if [ -f security-reports/trivy-fs-worker.sarif ]; then + echo "✅ Dependency vulnerability scan completed." + echo "" + echo "**Report artifact files:** trivy-fs-worker.sarif, trivy-fs-worker.html" + else + echo "❌ Dependency scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run secret scan with Gitleaks + run: | + GITLEAKS_VERSION=8.24.2 + curl -sSL -o gitleaks.tar.gz "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + tar -xzf gitleaks.tar.gz gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks detect \ + --source . \ + --redact \ + --verbose \ + --report-format json \ + --report-path security-reports/gitleaks-worker.json \ + --exit-code 0 + + - name: Generate Secret Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + json_path = pathlib.Path("security-reports/gitleaks-worker.json") + html_path = pathlib.Path("security-reports/gitleaks-worker.html") + title = "Secret Scan (Gitleaks) - Worker Service" + findings = [] + error = None + try: + text = json_path.read_text(encoding="utf-8").strip() + data = json.loads(text) if text else [] + if isinstance(data, list): + findings = data + elif isinstance(data, dict): + findings = data.get("findings") or data.get("results") or [] + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source JSON file: {html.escape(str(json_path))}

" + ] + if error: + parts.append(f"

Could not parse Gitleaks report: {html.escape(error)}

") + elif not findings: + parts.append("

No secrets were reported.

") + else: + parts.append(f"

Total findings: {len(findings)}

") + parts.append("") + for item in findings: + rule = item.get("RuleID") or item.get("rule") or "" + desc = item.get("Description") or item.get("description") or "" + file = item.get("File") or item.get("file") or "" + line = item.get("StartLine") or item.get("line") or "" + commit = item.get("Commit") or item.get("commit") or "" + parts.append("" + "".join(f"" for x in (rule, desc, file, line, commit)) + "") + parts.append("
RuleDescriptionFileLineCommit
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Secret Scan Summary + if: always() + run: | + { + echo "## Secret Scan (Gitleaks) - Worker Service" + echo "" + if [ -f security-reports/gitleaks-worker.json ]; then + echo "✅ Secret scan completed." + echo "" + echo "**Report artifact files:** gitleaks-worker.json, gitleaks-worker.html" + else + echo "❌ Secret scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Set version tag + run: | + if [[ "${GITHUB_REF##*/}" == "main" ]]; then + echo "TAG=v2" >> $GITHUB_ENV + else + echo "TAG=v1" >> $GITHUB_ENV + fi + + - name: Build Docker image + run: | + REGISTRY=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev + docker build -t worker-service:${TAG} -t $REGISTRY:worker-${TAG} ./worker + + - name: Security scan with Trivy + uses: aquasecurity/trivy-action@v0.20.0 + with: + scan-type: image + image-ref: worker-service:${{ env.TAG }} + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: '0' + format: sarif + output: security-reports/trivy-image-worker.sarif + + - name: Generate Container Image Scan HTML Report + if: always() + run: | + python - <<'PY' + import json, html, pathlib, datetime + sarif_path = pathlib.Path("security-reports/trivy-image-worker.sarif") + html_path = pathlib.Path("security-reports/trivy-image-worker.html") + title = "Container Image Scan (Trivy Image) - Worker Service" + rows = [] + error = None + try: + data = json.loads(sarif_path.read_text(encoding="utf-8")) + runs = data.get("runs", []) + for run in runs: + rules = {} + for rule in run.get("tool", {}).get("driver", {}).get("rules", []): + rid = rule.get("id", "") + rules[rid] = rule + for result in run.get("results", []): + rid = result.get("ruleId", "") + rule = rules.get(rid, {}) + level = result.get("level") or rule.get("defaultConfiguration", {}).get("level", "") + msg = result.get("message", {}).get("text", "") + loc = "" + locations = result.get("locations", []) + if locations: + phys = locations[0].get("physicalLocation", {}) + uri = phys.get("artifactLocation", {}).get("uri", "") + region = phys.get("region", {}) + line = region.get("startLine", "") + loc = f"{uri}:{line}" if line else uri + rows.append((level, rid, msg, loc)) + except Exception as exc: + error = str(exc) + now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + parts = [ + "", + f"{html.escape(title)}", + "", + "", + f"

{html.escape(title)}

", + f"

Generated at {html.escape(now)}

", + f"

Source SARIF file: {html.escape(str(sarif_path))}

" + ] + if error: + parts.append(f"

Could not parse SARIF report: {html.escape(error)}

") + elif not rows: + parts.append("

No findings were reported.

") + else: + parts.append(f"

Total findings: {len(rows)}

") + parts.append("") + for level, rid, msg, loc in rows: + parts.append("" + "".join(f"" for x in (level, rid, msg, loc)) + "") + parts.append("
LevelRule IDMessageLocation
{html.escape(str(x))}
") + parts.append("") + html_path.write_text("\n".join(parts), encoding="utf-8") + print(f"Generated {html_path}") + PY + + - name: Publish Container Image Scan Summary + if: always() + run: | + { + echo "## Container Image Scan (Trivy Image) - Worker Service" + echo "" + if [ -f security-reports/trivy-image-worker.sarif ]; then + echo "✅ Container image scan completed." + echo "" + echo "**Report artifact files:** trivy-image-worker.sarif, trivy-image-worker.html" + else + echo "❌ Container image scan report was not generated." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: DAST note for worker service + run: | + echo "Worker is a background service without an HTTP endpoint, so OWASP ZAP DAST is not applicable to this service." + echo "DAST is implemented for the vote and result web services." + + - name: Publish DAST Summary + if: always() + run: | + { + echo "## DAST (OWASP ZAP) - Worker Service" + echo "" + echo "ℹ️ DAST is not applicable because the worker is a background service without an HTTP endpoint." + echo "" + echo "DAST is implemented for the vote and result web services." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Publish Consolidated Security Summary + if: always() + run: | + { + echo "# DevSecOps Security Reports - Worker Service" + echo "" + echo "| Security Control | Report File |" + echo "|---|---|" + echo "| SAST (Semgrep) | semgrep-worker.sarif, semgrep-worker.html |" + echo "| SCA / Dependency Scan (Trivy FS) | trivy-fs-worker.sarif, trivy-fs-worker.html |" + echo "| Secret Scan (Gitleaks) | gitleaks-worker.json, gitleaks-worker.html |" + echo "| Container Image Scan (Trivy Image) | trivy-image-worker.sarif, trivy-image-worker.html |" + echo "| DAST (OWASP ZAP) | Not applicable - background service |" + echo "" + echo "Human-readable HTML reports and machine-readable SARIF/JSON reports are uploaded as the **security-reports-worker** workflow artifact." + echo "" + echo "## Files generated" + echo "" + ls -lah security-reports || true + echo "" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Security Reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-reports-worker + path: security-reports/ + if-no-files-found: warn + + - name: Push image to ECR + run: | + REGISTRY=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev + docker push $REGISTRY:worker-${TAG} + + - name: Configure kubectl + run: | + aws eks update-kubeconfig \ + --region ap-southeast-1 \ + --name bq-eks-cluster-dev + + - name: Deploy application + run: | + kubectl apply -f k8s-specifications/ + + - name: Update deployment + run: | + kubectl set image deployment/worker \ + worker=${{ steps.login-ecr.outputs.registry }}/bq-eks-repo-dev:worker-${TAG} + + - name: Wait for rollout + run: | + kubectl rollout status deployment/worker + diff --git a/.github/workflows/call-docker-build-worker.yaml b/.github/workflows/call-docker-build-worker.yaml deleted file mode 100644 index 5abfb6bc9c..0000000000 --- a/.github/workflows/call-docker-build-worker.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: Build Worker -# template source: https://github.com/dockersamples/.github/blob/main/templates/call-docker-build.yaml - -on: - # we want pull requests so we can build(test) but not push to image registry - push: - branches: - - 'main' - # only build when important files change - paths: - - 'worker/**' - - '.github/workflows/call-docker-build-worker.yaml' - pull_request: - branches: - - 'main' - # only build when important files change - paths: - - 'worker/**' - - '.github/workflows/call-docker-build-worker.yaml' - -jobs: - call-docker-build: - - name: Worker Call Docker Build - - uses: dockersamples/.github/.github/workflows/reusable-docker-build.yaml@main - - permissions: - contents: read - packages: write # needed to push docker image to ghcr.io - pull-requests: write # needed to create and update comments in PRs - - secrets: - - # Only needed if with:dockerhub-enable is true below - dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} - - # Only needed if with:dockerhub-enable is true below - dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} - - with: - - ### REQUIRED - ### ENABLE ONE OR BOTH REGISTRIES - ### tell docker where to push. - ### NOTE if Docker Hub is set to true, you must set secrets above and also add account/repo/tags below - dockerhub-enable: true - ghcr-enable: true - - ### REQUIRED - ### A list of the account/repo names for docker build. List should match what's enabled above - ### defaults to: - image-names: | - ghcr.io/dockersamples/example-voting-app-worker - dockersamples/examplevotingapp_worker - - ### REQUIRED set rules for tagging images, based on special action syntax: - ### https://github.com/docker/metadata-action#tags-input - ### defaults to: - tag-rules: | - type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=ref,event=pr - - ### path to where docker should copy files into image - ### defaults to root of repository (.) - context: worker - - ### Dockerfile alternate name. Default is Dockerfile (relative to context path) - # file: Containerfile - - ### build stage to target, defaults to empty, which builds to last stage in Dockerfile - # target: - - ### platforms to build for, defaults to linux/amd64 - ### other options: linux/amd64,linux/arm64,linux/arm/v7 - # FIXME worker arm/v7 support doesn't build in .net core 3.1 with QEMU - # a fix would likely run the .net build on amd64 but with a target of arm/v7 - platforms: linux/amd64,linux/arm64,linux/arm/v7 - - ### Create a PR comment with image tags and labels - ### defaults to false - # comment-enable: false diff --git a/build_and_push.sh b/build_and_push.sh new file mode 100755 index 0000000000..6bc90ffdfe --- /dev/null +++ b/build_and_push.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# ECR registry URI +REGISTRY=255945442255.dkr.ecr.ap-southeast-1.amazonaws.com/mightycapstonevoting + +# Authenticate Docker to ECR +aws ecr get-login-password --region ap-southeast-1 \ + | docker login --username AWS --password-stdin 255945442255.dkr.ecr.ap-southeast-1.amazonaws.com + +# Build & push Vote service +docker build -t $REGISTRY:vote-v1 ./vote +docker build -t $REGISTRY:vote-v2 ./vote +docker push $REGISTRY:vote-v1 +docker push $REGISTRY:vote-v2 + +# Build & push Result service +docker build -t $REGISTRY:result-v1 ./result +docker build -t $REGISTRY:result-v2 ./result +docker push $REGISTRY:result-v1 +docker push $REGISTRY:result-v2 + +# Build & push Worker service +docker build -t $REGISTRY:worker-v1 ./worker +docker build -t $REGISTRY:worker-v2 ./worker +docker push $REGISTRY:worker-v1 +docker push $REGISTRY:worker-v2 + +echo "✅ All images built and pushed to ECR" diff --git a/README.md b/docs/App Notes/README.md similarity index 100% rename from README.md rename to docs/App Notes/README.md diff --git a/architecture.excalidraw.png b/docs/App Notes/architecture.excalidraw.png similarity index 100% rename from architecture.excalidraw.png rename to docs/App Notes/architecture.excalidraw.png diff --git a/docs/ArchitectureDiagram.md b/docs/ArchitectureDiagram.md new file mode 100644 index 0000000000..6caaafc068 --- /dev/null +++ b/docs/ArchitectureDiagram.md @@ -0,0 +1,3 @@ +The following picture shows the complete CI/CD Pipleine for this implemented project + +![alt text](image.png) \ No newline at end of file diff --git a/docs/ard/0001-mightcapstone.md b/docs/ard/0001-mightcapstone.md new file mode 100644 index 0000000000..8efbecbd15 --- /dev/null +++ b/docs/ard/0001-mightcapstone.md @@ -0,0 +1,3 @@ +Assignment 15 + +![alt text](image.png) \ No newline at end of file diff --git a/docs/ard/image.png b/docs/ard/image.png new file mode 100644 index 0000000000..15551ef87c Binary files /dev/null and b/docs/ard/image.png differ diff --git a/docs/image.png b/docs/image.png new file mode 100644 index 0000000000..3aedecb616 Binary files /dev/null and b/docs/image.png differ diff --git a/k8s-specifications/db-deployment.yaml b/k8s-specifications/db-deployment.yaml index bc94ca7368..8aeb8dd8e4 100644 --- a/k8s-specifications/db-deployment.yaml +++ b/k8s-specifications/db-deployment.yaml @@ -1,9 +1,10 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: db-deployment + # namespace: mightycapstone labels: app: db - name: db spec: replicas: 1 selector: @@ -15,19 +16,21 @@ spec: app: db spec: containers: - - image: postgres:15-alpine - name: postgres - env: - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - value: postgres + - name: postgres + image: postgres:15-alpine ports: - containerPort: 5432 name: postgres + env: + - name: POSTGRES_USER + value: myuser + - name: POSTGRES_PASSWORD + value: mypassword + - name: POSTGRES_DB + value: mydatabase volumeMounts: - mountPath: /var/lib/postgresql/data - name: db-data + name: postgres-data volumes: - - name: db-data - emptyDir: {} + - name: postgres-data + emptyDir: {} # ephemeral storage; replace with PVC for persistence diff --git a/k8s-specifications/db-service.yaml b/k8s-specifications/db-service.yaml index 104f1e8268..8d8b3e566e 100644 --- a/k8s-specifications/db-service.yaml +++ b/k8s-specifications/db-service.yaml @@ -1,15 +1,16 @@ apiVersion: v1 kind: Service metadata: + name: db-service + # namespace: mightycapstone # 👈 ensure it matches your namespace labels: app: db - name: db spec: type: ClusterIP - ports: - - name: "db-service" - port: 5432 - targetPort: 5432 selector: app: db - + ports: + - name: postgres + port: 5432 # service port inside cluster + targetPort: 5432 # container port from db-deployment + diff --git a/k8s-specifications/redis-deployment.yaml b/k8s-specifications/redis-deployment.yaml index 24aa52135f..9feb2a23c7 100644 --- a/k8s-specifications/redis-deployment.yaml +++ b/k8s-specifications/redis-deployment.yaml @@ -1,9 +1,10 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: redis-deployment + # namespace: mightycapstone labels: app: redis - name: redis spec: replicas: 1 selector: @@ -15,8 +16,8 @@ spec: app: redis spec: containers: - - image: redis:alpine - name: redis + - name: redis + image: redis:alpine ports: - containerPort: 6379 name: redis @@ -25,4 +26,4 @@ spec: name: redis-data volumes: - name: redis-data - emptyDir: {} + emptyDir: {} # ephemeral storage; replace with PVC for persistence diff --git a/k8s-specifications/redis-service.yaml b/k8s-specifications/redis-service.yaml index 809d31d875..f3528cca2a 100644 --- a/k8s-specifications/redis-service.yaml +++ b/k8s-specifications/redis-service.yaml @@ -1,15 +1,16 @@ apiVersion: v1 kind: Service metadata: + name: redis-service + # namespace: mightycapstone labels: app: redis - name: redis spec: type: ClusterIP + selector: + app: redis ports: - - name: "redis-service" + - name: redis port: 6379 targetPort: 6379 - selector: - app: redis - + diff --git a/k8s-specifications/result-v1-deployment.yaml b/k8s-specifications/result-v1-deployment.yaml new file mode 100644 index 0000000000..0ee42af74b --- /dev/null +++ b/k8s-specifications/result-v1-deployment.yaml @@ -0,0 +1,40 @@ +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: result-v1-deployment +# namespace: mightycapstone +# # labels: +# app: result-v1 +# spec: +# replicas: 1 +# selector: +# matchLabels: +# app: result-v1 +# template: +# metadata: +# labels: +# app: result-v1 +# spec: +# containers: +# - name: result +# image: 255945442255.dkr.ecr.ap-southeast-1.amazonaws.com/mightycapstonevoting:result-v1 +# ports: +# - containerPort: 80 +# name: http +# env: +# - name: REDIS_HOST +# value: redis-service +# - name: DB_HOST +# value: db-service +# livenessProbe: +# httpGet: +# path: / +# port: 80 +# initialDelaySeconds: 20 +# periodSeconds: 10 +# readinessProbe: +# httpGet: +# path: / +# port: 80 +# initialDelaySeconds: 30 +# periodSeconds: 10 diff --git a/k8s-specifications/result-v1-service.yaml b/k8s-specifications/result-v1-service.yaml new file mode 100644 index 0000000000..c035463a3d --- /dev/null +++ b/k8s-specifications/result-v1-service.yaml @@ -0,0 +1,16 @@ +# apiVersion: v1 +# kind: Service +# metadata: +# name: result-v1-service +# namespace: mightycapstone +# labels: +# app: result-v1 +# spec: +# type: LoadBalancer # 👈 AWS ELB will be provisioned +# selector: +# app: result-v1 +# ports: +# - name: http +# port: 8081 # external port exposed by ELB +# targetPort: 80 # container port inside result pod + diff --git a/k8s-specifications/result-v2-deployment.yaml b/k8s-specifications/result-v2-deployment.yaml new file mode 100644 index 0000000000..7700e7f8a5 --- /dev/null +++ b/k8s-specifications/result-v2-deployment.yaml @@ -0,0 +1,40 @@ +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: result-v2-deployment +# namespace: mightycapstone +# labels: +# app: result-v2 +# spec: +# replicas: 1 +# selector: +# matchLabels: +# app: result-v2 +# template: +# metadata: +# labels: +# app: result-v2 +# spec: +# containers: +# - name: result +# image: 255945442255.dkr.ecr.ap-southeast-1.amazonaws.com/mightycapstonevoting:result-v2 +# ports: +# - containerPort: 80 +# name: http +# env: +# - name: REDIS_HOST +# value: redis-service +# - name: DB_HOST +# value: db-service +# livenessProbe: +# httpGet: +# path: / +# port: 80 +# initialDelaySeconds: 20 +# periodSeconds: 10 +# readinessProbe: +# httpGet: +# path: / +# port: 80 +# initialDelaySeconds: 30 +# periodSeconds: 10 diff --git a/k8s-specifications/vote-deployment.yaml b/k8s-specifications/vote-deployment.yaml index 165a9478f8..4023633789 100644 --- a/k8s-specifications/vote-deployment.yaml +++ b/k8s-specifications/vote-deployment.yaml @@ -1,11 +1,12 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: vote-deployment + # namespace: mightycapstone labels: app: vote - name: vote spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: vote @@ -15,8 +16,13 @@ spec: app: vote spec: containers: - - image: dockersamples/examplevotingapp_vote - name: vote + - name: vote + image: 255945442255.dkr.ecr.ap-southeast-1.amazonaws.com/mightycapstonevoting:vote-v2 ports: - containerPort: 80 - name: vote + name: http + env: + - name: REDIS_HOST + value: redis-service + - name: DB_HOST + value: db-service diff --git a/k8s-specifications/vote-service.yaml b/k8s-specifications/vote-service.yaml index d7a05b5513..5138736a4e 100644 --- a/k8s-specifications/vote-service.yaml +++ b/k8s-specifications/vote-service.yaml @@ -1,16 +1,16 @@ apiVersion: v1 kind: Service metadata: + name: vote-service + # namespace: mightycapstone labels: app: vote - name: vote spec: - type: NodePort - ports: - - name: "vote-service" - port: 8080 - targetPort: 80 - nodePort: 31000 + type: LoadBalancer # 👈 no nodePort here selector: app: vote - + ports: + - name: http + port: 8080 # external port exposed by ELB + targetPort: 80 # container port + diff --git a/k8s-specifications/worker-deployment.yaml b/k8s-specifications/worker-deployment.yaml index 9e35450aec..90ba0059cb 100644 --- a/k8s-specifications/worker-deployment.yaml +++ b/k8s-specifications/worker-deployment.yaml @@ -1,11 +1,12 @@ apiVersion: apps/v1 kind: Deployment metadata: + name: worker + # namespace: mightycapstone labels: app: worker - name: worker spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: worker @@ -15,5 +16,12 @@ spec: app: worker spec: containers: - - image: dockersamples/examplevotingapp_worker - name: worker + - name: worker + image: 255945442255.dkr.ecr.ap-southeast-1.amazonaws.com/mightycapstonevoting:worker-v2 + env: + - name: REDIS_HOST + value: redis-service + - name: DB_HOST + value: db-service + + diff --git a/result/Dockerfile b/result/Dockerfile index 33f0ba86fb..db836f2512 100644 --- a/result/Dockerfile +++ b/result/Dockerfile @@ -1,25 +1,14 @@ FROM node:18-slim -# add curl for healthcheck -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl tini && \ - rm -rf /var/lib/apt/lists/* - WORKDIR /usr/local/app -# have nodemon available for local dev use (file watching) -RUN npm install -g nodemon - COPY package*.json ./ - -RUN npm ci && \ - npm cache clean --force && \ - mv /usr/local/app/node_modules /node_modules +RUN npm ci && npm cache clean --force COPY . . ENV PORT=80 EXPOSE 80 -ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["node", "server.js"] + diff --git a/result/server.js b/result/server.js index 1c8593e7ee..ad3ce8c9ae 100644 --- a/result/server.js +++ b/result/server.js @@ -6,7 +6,7 @@ var express = require('express'), server = require('http').Server(app), io = require('socket.io')(server); -var port = process.env.PORT || 4000; +var port = process.env.PORT || 80; io.on('connection', function (socket) { @@ -18,7 +18,7 @@ io.on('connection', function (socket) { }); var pool = new Pool({ - connectionString: 'postgres://postgres:postgres@db/postgres' + connectionString: 'postgres://postgres:postgres@db-service/postgres' }); async.retry( diff --git a/worker/Dockerfile b/worker/Dockerfile index a3f92d7e94..7779344f51 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -8,20 +8,20 @@ # docker buildx build --platform "linux/arm64/v8" . # build compiles the program for the builder's local platform -FROM --platform=${BUILDPLATFORM} mcr.microsoft.com/dotnet/sdk:7.0 AS build -ARG TARGETPLATFORM +# Build stage +FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build ARG TARGETARCH -ARG BUILDPLATFORM -RUN echo "I am running on $BUILDPLATFORM, building for $TARGETPLATFORM" - WORKDIR /source + +# Copy project file and restore dependencies COPY *.csproj . RUN dotnet restore -a $TARGETARCH +# Copy source and publish COPY . . -RUN dotnet publish -c release -o /app -a $TARGETARCH --self-contained false --no-restore +RUN dotnet publish -c Release -o /app -a $TARGETARCH --self-contained false --no-restore -# app image +# Runtime stage FROM mcr.microsoft.com/dotnet/runtime:7.0 WORKDIR /app COPY --from=build /app .