diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 8a9820d..e03ca1a 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -16,9 +16,15 @@ services: - ministack ministack: - image: ministackorg/ministack + image: ministackorg/ministack:full ports: - "4566:4566" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + GLUE_DOCKER_IMAGE: ministack_glue_libs_4.0.0_image_01 + MINISTACK_ENDPOINT: http://ministack:4566 + S3_PERSIST: "1" restart: unless-stopped volumes: diff --git a/.github/workflows/publish-glue-image.yml b/.github/workflows/publish-glue-image.yml new file mode 100644 index 0000000..6d49e67 --- /dev/null +++ b/.github/workflows/publish-glue-image.yml @@ -0,0 +1,50 @@ +name: Build and Publish Custom MiniStack Glue Image + +on: + push: + branches: + - main + - develop + paths: + - 'docker/glue/**' + - '.github/workflows/publish-glue-image.yml' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/ministack_glue_libs_4.0.0_image_01 + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest + type=sha,format=short + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./docker/glue + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f8251eb..ff1db6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 --- +## [0.12.2] - 2026-06-24 + +### Added + +- Log forwarding module for MiniStack Glue/PySpark containers to stream output/error logs to local CloudWatch logs endpoint. +- GitHub Actions workflow to build and publish the custom MiniStack Glue Docker image to GHCR. +- Auto-discovery mechanism for the emulated MiniStack host inside the Docker bridge network. + ## [0.12.1] - 2026-06-24 ### Added diff --git a/README.md b/README.md index 9696d55..a2ed98b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@

- Version 0.12.1 + Version 0.12.2 MIT License Next.js 16+ AI Providers @@ -282,13 +282,35 @@ pnpm start ### 6. Deploy to Local AWS (MiniStack) -1. Start MiniStack: `docker run -p 4566:4566 ministackorg/ministack` +1. Start MiniStack: `docker run -p 4566:4566 -v /var/run/docker.sock:/var/run/docker.sock -e GLUE_DOCKER_IMAGE=ghcr.io/dmux/openarchflow/ministack_glue_libs_4.0.0_image_01:latest -e S3_PERSIST=1 ministackorg/ministack:full` 2. Click the **πŸš€ Rocket** icon in the toolbar 3. Click **Test Connection** β€” you should see "Connected" 4. Click **Deploy All** β€” nodes deploy in sequence with live status badges 5. Click any deployed node β†’ **Open Console** to interact with the resource 6. Run a simulation β€” deployed nodes receive real traffic from the simulation engine +#### πŸ“ AWS Glue / PySpark Emulation & Live Logs + +To emulate AWS Glue ETL jobs, query them using Athena SQL, and inspect execution logs directly in the OpenArchFlow UI, configure MiniStack using the `:full` image along with our custom Spark logging container: + +1. **Start MiniStack with Docker Socket & Custom Image**: + ```bash + docker run -d -p 4566:4566 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e GLUE_DOCKER_IMAGE=ghcr.io/dmux/openarchflow/ministack_glue_libs_4.0.0_image_01:latest \ + -e S3_PERSIST=1 \ + ministackorg/ministack:full + ``` + > [!NOTE] + > The `:full` tag of MiniStack is **required** to use the **Athena Query** tab. The default image does not contain the native DuckDB engine required to query real S3 files and returns static mocked data. + + *(Alternatively, use the provided `.devcontainer/docker-compose.yml` to spin up MiniStack automatically with these settings).* + +2. **Run PySpark Jobs**: + * Open the **Glue Studio** panel on a deployed Glue Catalog node. + * Start your job in the **Runs** tab. + * Enable **● Live** to stream JVM and Spark logs directly to the dashboard interface! + ### 7. Export Diagram - Click **Actions** β†’ **Export as PNG** diff --git a/docker/glue/Dockerfile b/docker/glue/Dockerfile new file mode 100644 index 0000000..163e257 --- /dev/null +++ b/docker/glue/Dockerfile @@ -0,0 +1,14 @@ +FROM amazon/aws-glue-libs:glue_libs_4.0.0_image_01 + +USER root + +# Copy forwarder script and entrypoint wrapper +COPY forward_logs.py /opt/glue/bin/forward_logs.py +COPY entrypoint.sh /opt/glue/bin/entrypoint.sh + +RUN chmod +x /opt/glue/bin/forward_logs.py /opt/glue/bin/entrypoint.sh + +# Revert to the default non-root user of aws-glue-libs +USER glue_user + +ENTRYPOINT ["/opt/glue/bin/entrypoint.sh"] diff --git a/docker/glue/entrypoint.sh b/docker/glue/entrypoint.sh new file mode 100644 index 0000000..d6c55a0 --- /dev/null +++ b/docker/glue/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +# Ensure the Spark bin directories are in the PATH +export PATH="/home/glue_user/spark/bin:/home/glue_user/maven/bin:${PATH}" + +# Run the command passed by MiniStack (typically spark-submit) +# and pipe both stdout and stderr through the python log forwarder. +exec "$@" 2>&1 | python3 /opt/glue/bin/forward_logs.py diff --git a/docker/glue/forward_logs.py b/docker/glue/forward_logs.py new file mode 100644 index 0000000..438447a --- /dev/null +++ b/docker/glue/forward_logs.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import sys +import os +import time +import urllib.request +import boto3 + +# Discover the working MiniStack endpoint from the sibling container +endpoints = [ + os.environ.get("MINISTACK_ENDPOINT"), + "http://host.docker.internal:4566", + "http://172.17.0.1:4566", + "http://ministack:4566", + "http://localhost:4566" +] + +# Filter valid and unique endpoints preserving order +endpoints = [e for e in endpoints if e] +seen = set() +endpoints = [e for e in endpoints if not (e in seen or seen.add(e))] + +endpoint = "http://host.docker.internal:4566" # fallback default +for ep in endpoints: + try: + # Test connection + urllib.request.urlopen(f"{ep}/", timeout=1) + endpoint = ep + sys.stderr.write(f"[log-forwarder] Successfully connected to MiniStack at: {endpoint}\n") + sys.stderr.flush() + break + except Exception: + continue + +log_group = os.environ.get("GLUE_LOG_GROUP", "/aws-glue/jobs/output") +log_stream = os.environ.get("GLUE_LOG_STREAM", "spark-stdout-stream") + +# Set up boto3 client pointing to discovered endpoint +client = boto3.client( + "logs", + endpoint_url=endpoint, + region_name=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", "mock"), + aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", "mock") +) + +# Ensure group and stream exist +try: + client.create_log_group(logGroupName=log_group) +except Exception: + pass + +try: + client.create_log_stream(logGroupName=log_group, logStreamName=log_stream) +except Exception: + pass + +# Pipe stdin to stdout and forward to CloudWatch Logs +try: + for line in sys.stdin: + sys.stdout.write(line) + sys.stdout.flush() + + stripped = line.strip() + if stripped: + try: + client.put_log_events( + logGroupName=log_group, + logStreamName=log_stream, + logEvents=[ + { + "timestamp": int(round(time.time() * 1000)), + "message": stripped + } + ] + ) + except Exception as e: + # Log send errors to stderr but don't crash + sys.stderr.write(f"\n[log-forwarder-send-error] {str(e)}\n") + sys.stderr.flush() +except KeyboardInterrupt: + pass +except Exception as e: + sys.stderr.write(f"\n[log-forwarder-error] {str(e)}\n") + sys.stderr.flush() diff --git a/package.json b/package.json index 76099fc..b67f9b3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "open-arch-flow", - "version": "0.12.1", + "version": "0.12.2", "private": true, "packageManager": "pnpm@11.1.2", "author": "Rafael Sales ", @@ -26,6 +26,7 @@ }, "dependencies": { "@aws-sdk/client-api-gateway": "^3.1073.0", + "@aws-sdk/client-athena": "^3.1075.0", "@aws-sdk/client-bedrock": "^3.1073.0", "@aws-sdk/client-bedrock-runtime": "^3.1073.0", "@aws-sdk/client-cloudfront": "^3.1073.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 110eb54..9a6f203 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@aws-sdk/client-api-gateway': specifier: ^3.1073.0 version: 3.1073.0 + '@aws-sdk/client-athena': + specifier: ^3.1075.0 + version: 3.1075.0 '@aws-sdk/client-bedrock': specifier: ^3.1073.0 version: 3.1073.0 @@ -323,6 +326,10 @@ packages: resolution: {integrity: sha512-mex6z2epRBR33Qa8YO6o3U7DCvpnT/NbtwPJMV9sntOucImJwh47xORxiH8gvmctw66iTRR4VDUAE6QTzLSIDw==} engines: {node: '>=20.0.0'} + '@aws-sdk/client-athena@3.1075.0': + resolution: {integrity: sha512-m5HnVSYGL0t+rB7Mr2cxKgjc9hN1DTtIvn39st4iaWOL3ku5sEqlqpmsXHMphxzbid1yKvUo1A68OAY3/ljByQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-bedrock-runtime@3.1073.0': resolution: {integrity: sha512-Vecj8r9/KIh/Nu9T7CRoCw5EBqnmAa9Q+Iwi5J5Mr0IEBMH6KUoOgAjayfyEZjvvZTllLJ2dOAx5cYeIz8QD6A==} engines: {node: '>=20.0.0'} @@ -587,10 +594,6 @@ packages: resolution: {integrity: sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==} engines: {node: '>=20.0.0'} - '@aws-sdk/types@3.973.8': - resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/util-locate-window@3.965.5': resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==} engines: {node: '>=20.0.0'} @@ -4285,7 +4288,7 @@ snapshots: '@aws-crypto/crc32@5.2.0': dependencies: '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 + '@aws-sdk/types': 3.973.13 tslib: 2.8.1 '@aws-crypto/crc32c@5.2.0': @@ -4298,7 +4301,7 @@ snapshots: dependencies: '@aws-crypto/supports-web-crypto': 5.2.0 '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 + '@aws-sdk/types': 3.973.13 '@aws-sdk/util-locate-window': 3.965.5 '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 @@ -4334,7 +4337,7 @@ snapshots: '@aws-crypto/crc32': 5.2.0 '@aws-crypto/crc32c': 5.2.0 '@aws-crypto/util': 5.2.0 - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/types': 4.15.0 @@ -4354,6 +4357,19 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 + '@aws-sdk/client-athena@3.1075.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.974.23 + '@aws-sdk/credential-provider-node': 3.972.58 + '@aws-sdk/types': 3.973.13 + '@smithy/core': 3.25.1 + '@smithy/fetch-http-handler': 5.5.1 + '@smithy/node-http-handler': 4.8.1 + '@smithy/types': 4.15.0 + tslib: 2.8.1 + '@aws-sdk/client-bedrock-runtime@3.1073.0': dependencies: '@aws-crypto/sha256-browser': 5.2.0 @@ -4757,7 +4773,7 @@ snapshots: '@aws-sdk/credential-provider-env@3.972.48': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/types': 4.15.0 @@ -4773,7 +4789,7 @@ snapshots: '@aws-sdk/credential-provider-http@3.972.50': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/fetch-http-handler': 5.5.1 @@ -4793,13 +4809,13 @@ snapshots: '@aws-sdk/credential-provider-ini@3.972.55': dependencies: - '@aws-sdk/core': 3.974.22 - '@aws-sdk/credential-provider-env': 3.972.48 - '@aws-sdk/credential-provider-http': 3.972.50 + '@aws-sdk/core': 3.974.23 + '@aws-sdk/credential-provider-env': 3.972.49 + '@aws-sdk/credential-provider-http': 3.972.51 '@aws-sdk/credential-provider-login': 3.972.54 - '@aws-sdk/credential-provider-process': 3.972.48 - '@aws-sdk/credential-provider-sso': 3.972.54 - '@aws-sdk/credential-provider-web-identity': 3.972.54 + '@aws-sdk/credential-provider-process': 3.972.49 + '@aws-sdk/credential-provider-sso': 3.972.55 + '@aws-sdk/credential-provider-web-identity': 3.972.55 '@aws-sdk/nested-clients': 3.997.22 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 @@ -4825,8 +4841,8 @@ snapshots: '@aws-sdk/credential-provider-login@3.972.54': dependencies: - '@aws-sdk/core': 3.974.22 - '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/core': 3.974.23 + '@aws-sdk/nested-clients': 3.997.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/types': 4.15.0 @@ -4871,7 +4887,7 @@ snapshots: '@aws-sdk/credential-provider-process@3.972.48': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/types': 4.15.0 @@ -4887,7 +4903,7 @@ snapshots: '@aws-sdk/credential-provider-sso@3.972.54': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/nested-clients': 3.997.22 '@aws-sdk/token-providers': 3.1071.0 '@aws-sdk/types': 3.973.13 @@ -4907,7 +4923,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.972.54': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/nested-clients': 3.997.22 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 @@ -4925,7 +4941,7 @@ snapshots: '@aws-sdk/dynamodb-codec@3.973.22': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@smithy/core': 3.25.1 '@smithy/types': 4.15.0 tslib: 2.8.1 @@ -4971,7 +4987,7 @@ snapshots: '@aws-sdk/middleware-sdk-ec2@3.972.36': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/signature-v4': 5.5.1 @@ -4980,7 +4996,7 @@ snapshots: '@aws-sdk/middleware-sdk-rds@3.972.36': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/signature-v4': 5.5.1 @@ -4995,7 +5011,7 @@ snapshots: '@aws-sdk/middleware-sdk-s3@3.972.53': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/signature-v4-multi-region': 3.996.35 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 @@ -5011,7 +5027,7 @@ snapshots: '@aws-sdk/middleware-websocket@3.972.30': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/fetch-http-handler': 5.5.1 @@ -5023,7 +5039,7 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 5.2.0 '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/signature-v4-multi-region': 3.996.35 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 @@ -5054,8 +5070,8 @@ snapshots: '@aws-sdk/token-providers@3.1071.0': dependencies: - '@aws-sdk/core': 3.974.22 - '@aws-sdk/nested-clients': 3.997.22 + '@aws-sdk/core': 3.974.23 + '@aws-sdk/nested-clients': 3.997.23 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 '@smithy/types': 4.15.0 @@ -5063,7 +5079,7 @@ snapshots: '@aws-sdk/token-providers@3.1073.0': dependencies: - '@aws-sdk/core': 3.974.22 + '@aws-sdk/core': 3.974.23 '@aws-sdk/nested-clients': 3.997.22 '@aws-sdk/types': 3.973.13 '@smithy/core': 3.25.1 @@ -5084,11 +5100,6 @@ snapshots: '@smithy/types': 4.15.0 tslib: 2.8.1 - '@aws-sdk/types@3.973.8': - dependencies: - '@smithy/types': 4.15.0 - tslib: 2.8.1 - '@aws-sdk/util-locate-window@3.965.5': dependencies: tslib: 2.8.1 diff --git a/src/components/ministack/GlueStudioPanel.tsx b/src/components/ministack/GlueStudioPanel.tsx index cb2a12b..ca1782f 100644 --- a/src/components/ministack/GlueStudioPanel.tsx +++ b/src/components/ministack/GlueStudioPanel.tsx @@ -3,7 +3,7 @@ import React, { useState, useEffect, useMemo } from "react"; import { createPortal } from "react-dom"; import { motion, AnimatePresence } from "framer-motion"; -import { X, Database, Table2, Cog, Activity, AlertTriangle } from "lucide-react"; +import { X, Database, Table2, Cog, Activity, AlertTriangle, Maximize2, Minimize2, Search } from "lucide-react"; import { ArchitectureServiceAWSGlue } from "aws-react-icons"; import { cn } from "@/lib/utils"; import { useDiagramStore, type AppNode, type GlueJobConfig } from "@/lib/store"; @@ -13,10 +13,11 @@ import { CatalogTab } from "./glue/CatalogTab"; import { TablesTab } from "./glue/TablesTab"; import { JobsTab } from "./glue/JobsTab"; import { RunsTab } from "./glue/RunsTab"; +import { AthenaQueryTab } from "./glue/AthenaQueryTab"; const GLUE_ACCENT = "#8C4FFF"; -type StudioTab = "catalog" | "tables" | "jobs" | "runs"; +type StudioTab = "catalog" | "tables" | "jobs" | "runs" | "query"; interface GlueStudioPanelProps { isOpen: boolean; @@ -35,9 +36,19 @@ export default function GlueStudioPanel({ isOpen, onClose, nodes }: GlueStudioPa const [activeDatabase, setActiveDatabase] = useState(""); const [activeJob, setActiveJob] = useState(""); const [reachable, setReachable] = useState(null); + const [isFullscreen, setIsFullscreen] = useState(false); useEffect(() => setMounted(true), []); + // Esc key exits fullscreen + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (e.key === "Escape" && isFullscreen) setIsFullscreen(false); + }; + window.addEventListener("keydown", handler); + return () => window.removeEventListener("keydown", handler); + }, [isFullscreen]); + // Probe the endpoint directly β€” connectivity is independent of the // `enabled` flag, which only flips after a MiniStack panel deploy. useEffect(() => { @@ -77,21 +88,28 @@ export default function GlueStudioPanel({ isOpen, onClose, nodes }: GlueStudioPa { id: "tables", label: "Tables", icon: }, { id: "jobs", label: "Jobs", icon: }, { id: "runs", label: "Runs", icon: }, + { id: "query", label: "Athena Query", icon: }, ]; const panel = ( {isOpen && ( {/* Header */} -

+
@@ -108,9 +126,18 @@ export default function GlueStudioPanel({ isOpen, onClose, nodes }: GlueStudioPa

- +
+ + +
{/* Node selector (when multiple Glue nodes) */} @@ -184,6 +211,9 @@ export default function GlueStudioPanel({ isOpen, onClose, nodes }: GlueStudioPa {activeTab === "runs" && ( )} + {activeTab === "query" && ( + + )} )}
diff --git a/src/components/ministack/glue/AthenaQueryTab.tsx b/src/components/ministack/glue/AthenaQueryTab.tsx new file mode 100644 index 0000000..f6d8870 --- /dev/null +++ b/src/components/ministack/glue/AthenaQueryTab.tsx @@ -0,0 +1,341 @@ +"use client"; + +import React, { useState, useEffect, useRef } from "react"; +import Editor, { loader, useMonaco } from "@monaco-editor/react"; +import { useTheme } from "next-themes"; +import { Play, Loader2, Database, AlertCircle, Copy, Check, Download } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; +import { cn } from "@/lib/utils"; +import { toast } from "sonner"; +import type { MiniStackConfig } from "@/lib/ministack/types"; +import { glueGetTables, type GlueTableInfo } from "@/lib/ministack/glue-actions"; +import { athenaRunQuery, type AthenaQueryResult } from "@/lib/ministack/athena-actions"; + +loader.config({ + paths: { vs: "https://cdn.jsdelivr.net/npm/monaco-editor@0.52.0/min/vs" }, +}); + +interface AthenaQueryTabProps { + config: MiniStackConfig; + databaseName: string; +} + +let sqlSnippetsRegistered = false; + +function registerSqlAutocompletion(monaco: any, tables: GlueTableInfo[]) { + // Always register keyword completions. Re-registers on tables change if needed, + // but to prevent multiples, we register dynamic completions based on the active closure. + if (sqlSnippetsRegistered) return; + sqlSnippetsRegistered = true; + + monaco.languages.registerCompletionItemProvider("sql", { + provideCompletionItems: (model: any, position: any) => { + const word = model.getWordUntilPosition(position); + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn, + }; + + const sqlKeywords = [ + "SELECT", "FROM", "WHERE", "GROUP BY", "ORDER BY", "HAVING", "LIMIT", + "JOIN", "LEFT JOIN", "RIGHT JOIN", "INNER JOIN", "ON", "AND", "OR", + "AS", "COUNT", "SUM", "AVG", "MIN", "MAX", "DISTINCT", "CREATE", "TABLE", + "SHOW", "DESCRIBE", "LIKE", "IN", "IS NULL", "IS NOT NULL" + ]; + + const suggestions: any[] = sqlKeywords.map((k) => ({ + label: k, + kind: monaco.languages.CompletionItemKind.Keyword, + insertText: k, + detail: "SQL Keyword", + range, + })); + + // Add tables and columns from the global state/closure + // We read window.athenaTables if set to allow dynamic updates without multiple registrations + const activeTables: GlueTableInfo[] = (window as any).athenaTables || tables || []; + + activeTables.forEach((t) => { + // Table suggestion + suggestions.push({ + label: t.name, + kind: monaco.languages.CompletionItemKind.Class, + insertText: t.name, + detail: `Glue Table (${t.columns.length} columns)`, + range, + }); + + // Column suggestions + t.columns.forEach((col) => { + suggestions.push({ + label: col.name, + kind: monaco.languages.CompletionItemKind.Field, + insertText: col.name, + detail: `${t.name}.${col.name} (${col.type})`, + range, + }); + }); + }); + + return { suggestions }; + }, + }); +} + +export function AthenaQueryTab({ config, databaseName }: AthenaQueryTabProps) { + const { resolvedTheme } = useTheme(); + const monaco = useMonaco(); + const [query, setQuery] = useState(""); + const [running, setRunning] = useState(false); + const [results, setResults] = useState(null); + const [error, setError] = useState(null); + const [tables, setTables] = useState([]); + const [copied, setCopied] = useState(false); + + // Load tables for autocompletion + useEffect(() => { + if (!databaseName) return; + glueGetTables(config, databaseName) + .then((tList) => { + setTables(tList); + (window as any).athenaTables = tList; // Store globally for autocomplete closure + if (tList.length > 0 && !query) { + // Provide a friendly default query + setQuery(`SELECT * FROM ${tList[0].name} LIMIT 10;`); + } + }) + .catch(() => {}); + }, [config, databaseName]); + + // Register Monaco SQL snippets + useEffect(() => { + if (!monaco) return; + registerSqlAutocompletion(monaco, tables); + }, [monaco, tables]); + + const handleRunQuery = async () => { + if (!query.trim() || !databaseName) return; + setRunning(true); + setError(null); + setResults(null); + try { + const res = await athenaRunQuery(config, databaseName, query); + setResults(res); + toast.success("Query executed successfully!"); + } catch (e) { + setError(e instanceof Error ? e.message : "Failed to execute query"); + toast.error("Query failed"); + } finally { + setRunning(false); + } + }; + + const handleCopy = () => { + if (!results) return; + const text = [ + results.columns.join("\t"), + ...results.rows.map((r) => results.columns.map((c) => r[c]).join("\t")) + ].join("\n"); + navigator.clipboard.writeText(text); + setCopied(true); + toast.success("Results copied as TSV"); + setTimeout(() => setCopied(false), 2000); + }; + + const handleDownloadCsv = () => { + if (!results) return; + const csvContent = [ + results.columns.map(c => `"${c.replace(/"/g, '""')}"`).join(","), + ...results.rows.map((r) => results.columns.map((c) => `"${(r[c] || "").replace(/"/g, '""')}"`).join(",")) + ].join("\n"); + + const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = `query_results_${Date.now()}.csv`; + link.click(); + URL.revokeObjectURL(url); + toast.success("CSV file downloaded!"); + }; + + return ( +
+ {/* Editor & Sidebar layout */} +
+ {/* SQL editor */} +
+
+ + {databaseName || "default"}.sql + + +
+
+ setQuery(val ?? "")} + theme={resolvedTheme === "dark" ? "vs-dark" : "vs"} + options={{ + fontSize: 12.5, + fontFamily: "'JetBrains Mono', 'Fira Code', 'Cascadia Code', monospace", + minimap: { enabled: false }, + lineNumbers: "on", + renderLineHighlight: "all", + wordWrap: "on", + padding: { top: 8, bottom: 8 }, + scrollbar: { verticalScrollbarSize: 5, horizontalScrollbarSize: 5 }, + quickSuggestions: { other: true, comments: false, strings: false }, + suggestOnTriggerCharacters: true, + }} + /> +
+
+ + {/* Database catalog tables list sidebar */} +
+
+ + Tables ({tables.length}) +
+ +
+ {tables.length === 0 && ( +

No tables found.

+ )} + {tables.map((t) => ( +
+ setQuery((q) => `${q.replace(/;?\s*$/, "")}\nSELECT * FROM ${t.name} LIMIT 10;`)} + title="Click to build select query" + > + πŸ“ {t.name} + +
+ {t.columns.map((col) => ( +
+ {col.name} + {col.type} +
+ ))} +
+
+ ))} +
+ +
+
+ + {/* Query output and results view */} +
+
+ Results + {results && ( +
+ + {results.rows.length} row{results.rows.length !== 1 ? "s" : ""} + + + +
+ )} +
+ +
+ {running && ( +
+ + Athena executing SQL query via DuckDB… +
+ )} + + {error && ( +
+ +
+

Query Execution Error

+ 
{error}
+
+
+ )} + + {!results && !error && !running && ( +
+

Execute an Athena SQL query above to query catalog tables.

+

Autocomplete includes catalog tables and column schemas dynamically.

+
+ )} + + {results && !error && ( + +
+ + + + {results.columns.map((c) => ( + + ))} + + + + {results.rows.length === 0 ? ( + + + + ) : ( + results.rows.map((row, rIdx) => ( + + {results.columns.map((c) => ( + + ))} + + )) + )} + +
+ {c} +
+ Query returned 0 rows. +
+ {row[c] !== "" ? row[c] : null} +
+
+ + + + )} +
+
+
+ ); +} diff --git a/src/components/ministack/glue/JobsTab.tsx b/src/components/ministack/glue/JobsTab.tsx index cf9ca9b..28d0564 100644 --- a/src/components/ministack/glue/JobsTab.tsx +++ b/src/components/ministack/glue/JobsTab.tsx @@ -1,7 +1,7 @@ "use client"; import React, { useState, useEffect, useCallback, useMemo } from "react"; -import { Cog, Plus, RefreshCw, Loader2, Trash2, Rocket, Play, ChevronLeft } from "lucide-react"; +import { Cog, Plus, RefreshCw, Loader2, Trash2, Rocket, Play, ChevronLeft, Copy, Check, Download, Monitor, Sun, Moon } from "lucide-react"; import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; import { cn } from "@/lib/utils"; @@ -11,7 +11,7 @@ import type { AppNode, GlueJobConfig } from "@/lib/store"; import { glueListJobs, glueUpsertJob, glueDeleteJob, glueUploadScript, glueStartJobRun, sanitizeGlueName, } from "@/lib/ministack/glue-actions"; -import PySparkEditor, { DEFAULT_PYSPARK_SCRIPT } from "./PySparkEditor"; +import PySparkEditor, { DEFAULT_PYSPARK_SCRIPT, type EditorThemeMode } from "./PySparkEditor"; const GLUE_VERSIONS = ["4.0", "3.0"]; const WORKER_TYPES = ["G.1X", "G.2X"]; @@ -34,6 +34,39 @@ export function JobsTab({ config, node, setNodeGlueConfig, onRunStarted }: JobsT const [editing, setEditing] = useState(null); const [deploying, setDeploying] = useState(false); const [running, setRunning] = useState(false); + const [copied, setCopied] = useState(false); + const [editorThemeMode, setEditorThemeMode] = useState("app"); + + const handleCopy = () => { + if (!editing) return; + navigator.clipboard.writeText(editing.pysparkCode); + setCopied(true); + toast.success("Script copied to clipboard!"); + setTimeout(() => setCopied(false), 2000); + }; + + const handleDownload = () => { + if (!editing) return; + const blob = new Blob([editing.pysparkCode], { type: "text/plain;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = `${editing.name || "etl_job"}.py`; + link.click(); + URL.revokeObjectURL(url); + toast.success("Script downloaded!"); + }; + + const updateArg = (key: string, val: string) => { + if (!editing) return; + const newArgs = { ...(editing.arguments ?? {}) }; + if (val.trim() === "") { + delete newArgs[key]; + } else { + newArgs[key] = val; + } + setEditing({ ...editing, arguments: newArgs }); + }; const load = useCallback(async () => { setLoading(true); @@ -76,6 +109,7 @@ export function JobsTab({ config, node, setNodeGlueConfig, onRunStarted }: JobsT glueVersion: editing.glueVersion, workerType: editing.workerType, numberOfWorkers: editing.numberOfWorkers, + arguments: editing.arguments, }); const saved = { ...editing, name }; persist(saved); @@ -163,11 +197,100 @@ export function JobsTab({ config, node, setNodeGlueConfig, onRunStarted }: JobsT onChange={(e) => setEditing({ ...editing, numberOfWorkers: Math.max(2, Number(e.target.value)) })} className="h-7 w-20 text-xs rounded border border-border bg-background px-2" /> - Spark runs on the amazon/aws-glue-libs image (Docker required). Iceberg: add --datalake-formats iceberg. + Docker required. Iceberg: add --datalake-formats iceberg. -
- setEditing({ ...editing, pysparkCode: v })} /> +
+
+ + updateArg("--additional-python-modules", e.target.value)} + placeholder="pandas,requests" + className="h-7 w-full text-[11px] rounded border border-border bg-background px-2 font-mono" + title="Additional Python modules (pip install)" + /> +
+
+ + +
+
+ + updateArg("--extra-py-files", e.target.value)} + placeholder="s3://bucket/utils.py" + className="h-7 w-full text-[11px] rounded border border-border bg-background px-2 font-mono" + title="Paths to extra Python files (e.g. on S3)" + /> +
+
+ +
+
+ script.py +
+
+ {( + [ + { mode: "app" as EditorThemeMode, icon: , title: "Sync with app theme" }, + { mode: "python-light" as EditorThemeMode, icon: , title: "Light theme" }, + { mode: "python-dark" as EditorThemeMode, icon: , title: "Dark theme" }, + ] as const + ).map(({ mode, icon, title }) => ( + + ))} +
+ + +
+
+
+ setEditing({ ...editing, pysparkCode: v })} + themeMode={editorThemeMode} + /> +
); diff --git a/src/components/ministack/glue/PySparkEditor.tsx b/src/components/ministack/glue/PySparkEditor.tsx index 3a0eef0..b68cef3 100644 --- a/src/components/ministack/glue/PySparkEditor.tsx +++ b/src/components/ministack/glue/PySparkEditor.tsx @@ -1,11 +1,18 @@ "use client"; -import React, { useEffect } from "react"; -import Editor, { useMonaco } from "@monaco-editor/react"; +import React, { useRef, useEffect } from "react"; +import Editor, { loader, useMonaco } from "@monaco-editor/react"; import { useTheme } from "next-themes"; +loader.config({ + paths: { vs: "https://cdn.jsdelivr.net/npm/monaco-editor@0.52.0/min/vs" }, +}); + // Default AWS Glue PySpark boilerplate (GlueContext + Job bookmarks). export const DEFAULT_PYSPARK_SCRIPT = `import sys +import uuid +from pyspark.sql import functions as F +from pyspark.sql.types import StringType from awsglue.transforms import * from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext @@ -19,32 +26,77 @@ spark = glueContext.spark_session job = Job(glueContext) job.init(args["JOB_NAME"], args) -# Read a table from the Glue Data Catalog -# source = glueContext.create_dynamic_frame.from_catalog( -# database="my_database", -# table_name="my_table", -# ) - -# Transform with Spark, then write back to S3 / the catalog -# source.toDF().show() -# glueContext.write_dynamic_frame.from_options( -# frame=source, -# connection_type="s3", -# connection_options={"path": "s3://my-bucket/output/"}, -# format="parquet", -# ) +row_count = 1000 +base_df = spark.range(0, row_count).withColumn("value", F.rand()) + +uuid_udf = F.udf(lambda: str(uuid.uuid4()), StringType()) +random_df = base_df.withColumn("id", uuid_udf()).select("id", "value") + +random_df.show(10, truncate=False) + +random_df.write.mode("overwrite").parquet( + "s3://openarchflow-glue-scripts/tables/table/" +) job.commit() `; +function registerPythonThemes(monaco: any) { + monaco.editor.defineTheme("python-dark", { + base: "vs-dark", + inherit: true, + rules: [ + { token: "keyword", foreground: "ffcb6b", fontStyle: "bold" }, + { token: "identifier", foreground: "82aaff" }, + { token: "string", foreground: "c3e88d" }, + { token: "number", foreground: "f78c6c" }, + { token: "comment", foreground: "546e7a", fontStyle: "italic" }, + { token: "operator", foreground: "89ddff" }, + { token: "delimiter.bracket", foreground: "89ddff" }, + ], + colors: { + "editor.background": "#0d1117", + "editor.foreground": "#c9d1d9", + "editorLineNumber.foreground": "#484f58", + "editorLineNumber.activeForeground": "#58a6ff", + "editor.selectionBackground": "#58a6ff33", + "editor.lineHighlightBackground": "#21262d55", + "editorCursor.foreground": "#58a6ff", + "editorGutter.background": "#0d1117", + }, + }); + + monaco.editor.defineTheme("python-light", { + base: "vs", + inherit: true, + rules: [ + { token: "keyword", foreground: "005cc5", fontStyle: "bold" }, + { token: "identifier", foreground: "032f62" }, + { token: "string", foreground: "22863a" }, + { token: "number", foreground: "005cc5" }, + { token: "comment", foreground: "6a737d", fontStyle: "italic" }, + { token: "operator", foreground: "d73a49" }, + ], + colors: { + "editor.background": "#f6f8fa", + "editor.foreground": "#24292e", + "editorLineNumber.foreground": "#57606a", + "editorLineNumber.activeForeground": "#0969da", + "editor.selectionBackground": "#0969da22", + "editor.lineHighlightBackground": "#eaeef2aa", + "editorCursor.foreground": "#0969da", + }, + }); +} + let snippetsRegistered = false; -function registerPySparkSnippets(monaco: NonNullable>) { +function registerPySparkSnippets(monaco: any) { if (snippetsRegistered) return; snippetsRegistered = true; monaco.languages.registerCompletionItemProvider("python", { - provideCompletionItems: (model, position) => { + provideCompletionItems: (model: any, position: any) => { const word = model.getWordUntilPosition(position); const range = { startLineNumber: position.lineNumber, @@ -80,6 +132,46 @@ function registerPySparkSnippets(monaco: NonNullable ({ @@ -95,11 +187,14 @@ function registerPySparkSnippets(monaco: NonNullable void; readOnly?: boolean; height?: string; + themeMode?: EditorThemeMode; } export default function PySparkEditor({ @@ -107,36 +202,66 @@ export default function PySparkEditor({ onChange, readOnly = false, height = "100%", + themeMode = "app", }: PySparkEditorProps) { const { resolvedTheme } = useTheme(); const monaco = useMonaco(); + const editorRef = useRef(null); + // Register themes and snippets when Monaco is loaded useEffect(() => { if (!monaco) return; + registerPythonThemes(monaco); registerPySparkSnippets(monaco); }, [monaco]); + // Sync theme when context theme changes + useEffect(() => { + if (!monaco) return; + const resolved = + themeMode === "app" + ? resolvedTheme === "dark" + ? "python-dark" + : "python-light" + : themeMode; + monaco.editor.setTheme(resolved); + }, [monaco, resolvedTheme, themeMode]); + + const initialTheme = + themeMode === "app" + ? resolvedTheme === "dark" + ? "python-dark" + : "python-light" + : themeMode; + return ( onChange?.(val ?? "")} + onMount={(editor) => { editorRef.current = editor; }} options={{ readOnly, - fontSize: 12, + fontSize: 13, fontFamily: "'JetBrains Mono', 'Fira Code', 'Cascadia Code', monospace", - minimap: { enabled: false }, + fontLigatures: true, + minimap: { enabled: true, scale: 1 }, + folding: true, lineNumbers: "on", wordWrap: "on", + renderLineHighlight: "all", scrollBeyondLastLine: false, - padding: { top: 8, bottom: 8 }, - overviewRulerLanes: 0, - renderLineHighlight: "gutter", - tabSize: 4, + smoothScrolling: true, + cursorBlinking: "smooth", + padding: { top: 12, bottom: 12 }, + overviewRulerBorder: false, + scrollbar: { verticalScrollbarSize: 6, horizontalScrollbarSize: 6 }, suggestOnTriggerCharacters: true, quickSuggestions: { other: true, comments: false, strings: false }, + bracketPairColorization: { enabled: true }, + tabSize: 4, }} /> ); diff --git a/src/components/ministack/glue/RunsTab.tsx b/src/components/ministack/glue/RunsTab.tsx index db7cf65..8ccaddc 100644 --- a/src/components/ministack/glue/RunsTab.tsx +++ b/src/components/ministack/glue/RunsTab.tsx @@ -1,7 +1,7 @@ "use client"; import React, { useState, useEffect, useCallback, useRef } from "react"; -import { Activity, RefreshCw, Loader2, Terminal, Copy } from "lucide-react"; +import { Activity, RefreshCw, Loader2, Terminal, Copy, Trash2 } from "lucide-react"; import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; import { cn } from "@/lib/utils"; @@ -11,7 +11,7 @@ import { glueListJobs, glueListJobRuns, glueGetJobRun, isTerminalRunState, type GlueJobRunInfo, type GlueJobRunState, } from "@/lib/ministack/glue-actions"; -import { cwlStreamEvents, cwlListGroups, type CwlLogEvent } from "@/lib/ministack/browser-actions"; +import { cwlStreamEvents, cwlListGroups, cwlClearGroup, type CwlLogEvent } from "@/lib/ministack/browser-actions"; const STATE_STYLES: Record = { SUCCEEDED: "bg-green-500/15 text-green-600 border-green-500/30", @@ -39,6 +39,16 @@ export function RunsTab({ config, activeJob, onSelectJob }: RunsTabProps) { const [streaming, setStreaming] = useState(false); const [allGroups, setAllGroups] = useState(null); const [selectedGroup, setSelectedGroup] = useState(""); // "" = all groups + const [autoscroll, setAutoscroll] = useState(true); + + const logEndRef = useRef(null); + + // Scroll to bottom when logs update and autoscroll is enabled + useEffect(() => { + if (autoscroll && logEndRef.current) { + logEndRef.current.scrollIntoView({ behavior: "smooth" }); + } + }, [logs, autoscroll]); // List every CloudWatch log group so the user can see/pick where MiniStack // writes Glue Spark logs (the naming is not guaranteed to be /aws-glue/*). @@ -87,6 +97,7 @@ export function RunsTab({ config, activeJob, onSelectJob }: RunsTabProps) { useEffect(() => { if (hasActiveRun) setStreaming(true); }, [hasActiveRun]); + const pollRef = useRef(null); useEffect(() => { if (!activeJob || !hasActiveRun) return; @@ -120,6 +131,28 @@ export function RunsTab({ config, activeJob, onSelectJob }: RunsTabProps) { const fmtTime = (sec?: number) => (sec === undefined ? "β€”" : `${sec}s`); + const handleClearAllRuns = () => { + setRuns([]); + toast.success("Cleared all runs from view"); + }; + + const handleClearIndividualRun = (runId: string) => { + setRuns((prev) => prev.filter((r) => r.id !== runId)); + toast.success("Execution cleared from view"); + }; + + const handleClearCloudWatchLogs = async () => { + const target = selectedGroup || "/aws-glue/jobs/output"; + try { + await cwlClearGroup(config, target); + setLogs([]); + toast.success(`Cleared CloudWatch Log Group: ${target}`); + await loadGroups(); + } catch (e) { + toast.error(e instanceof Error ? e.message : "Failed to clear CloudWatch logs"); + } + }; + return (
@@ -132,7 +165,18 @@ export function RunsTab({ config, activeJob, onSelectJob }: RunsTabProps) { {jobNames.length === 0 && } {jobNames.map((n) => )} - + )} +
@@ -142,13 +186,22 @@ export function RunsTab({ config, activeJob, onSelectJob }: RunsTabProps) {

{loading ? "Loading…" : "No runs yet β€” start one from the Jobs tab."}

)} {runs.map((r) => ( -
+
{r.id} - - {!isTerminalRunState(r.state) && } - {r.state} - +
+ + {!isTerminalRunState(r.state) && } + {r.state} + + +
Exec: {fmtTime(r.executionTimeSeconds)} @@ -186,6 +239,26 @@ export function RunsTab({ config, activeJob, onSelectJob }: RunsTabProps) {
+ +
))} +
diff --git a/src/lib/ministack/athena-actions.ts b/src/lib/ministack/athena-actions.ts new file mode 100644 index 0000000..744ead5 --- /dev/null +++ b/src/lib/ministack/athena-actions.ts @@ -0,0 +1,73 @@ +import { getAthenaClient } from "./client"; +import type { MiniStackConfig } from "./types"; +import { + StartQueryExecutionCommand, + GetQueryExecutionCommand, + GetQueryResultsCommand, +} from "@aws-sdk/client-athena"; + +export interface AthenaQueryResult { + columns: string[]; + rows: Record[]; +} + +export async function athenaRunQuery( + config: MiniStackConfig, + databaseName: string, + queryString: string, +): Promise { + const athena = getAthenaClient(config); + + // Athena requires OutputLocation to store result files in S3 + const OutputLocation = `s3://openarchflow-glue-scripts/query-results/`; + + const startRes = await athena.send(new StartQueryExecutionCommand({ + QueryString: queryString, + QueryExecutionContext: { Database: databaseName }, + ResultConfiguration: { OutputLocation }, + })); + + const queryExecutionId = startRes.QueryExecutionId; + if (!queryExecutionId) throw new Error("Failed to start query execution"); + + // Poll for query completion + let status = "RUNNING"; + let tries = 0; + while ((status === "RUNNING" || status === "QUEUED") && tries < 60) { + await new Promise((resolve) => setTimeout(resolve, 500)); + const statusRes = await athena.send(new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId })); + status = statusRes.QueryExecution?.Status?.State ?? "FAILED"; + if (status === "FAILED" || status === "CANCELLED") { + throw new Error(statusRes.QueryExecution?.Status?.StateChangeReason || "Query execution failed or cancelled"); + } + tries++; + } + + if (status !== "SUCCEEDED") { + throw new Error("Query timed out or failed to complete"); + } + + // Fetch results + const resultsRes = await athena.send(new GetQueryResultsCommand({ QueryExecutionId: queryExecutionId })); + const resultSet = resultsRes.ResultSet; + if (!resultSet || !resultSet.Rows || resultSet.Rows.length === 0) { + return { columns: [], rows: [] }; + } + + const rows = resultSet.Rows; + // First row is the headers/columns definition + const headerRow = rows[0].Data ?? []; + const columns = headerRow.map((d) => d.VarCharValue ?? ""); + + const records: Record[] = []; + for (let i = 1; i < rows.length; i++) { + const data = rows[i].Data ?? []; + const record: Record = {}; + for (let j = 0; j < columns.length; j++) { + record[columns[j]] = data[j]?.VarCharValue ?? ""; + } + records.push(record); + } + + return { columns, rows: records }; +} diff --git a/src/lib/ministack/browser-actions.ts b/src/lib/ministack/browser-actions.ts index f30bf33..a0817b9 100644 --- a/src/lib/ministack/browser-actions.ts +++ b/src/lib/ministack/browser-actions.ts @@ -66,6 +66,7 @@ import { CreateKeyCommand, ScheduleKeyDeletionCommand } from "@aws-sdk/client-km // CloudWatch Logs import { DescribeLogGroupsCommand, DescribeLogStreamsCommand, FilterLogEventsCommand, + DeleteLogGroupCommand, DeleteLogStreamCommand, CreateLogGroupCommand, } from "@aws-sdk/client-cloudwatch-logs"; // ── Browser-safe helpers ────────────────────────────────────────────────────── @@ -829,3 +830,18 @@ export function cwlStreamEvents( const id = setInterval(() => poll().catch(() => {}), 2000); return () => clearInterval(id); } + +export async function cwlClearGroup(config: MiniStackConfig, logGroupName: string): Promise { + const cwl = getCloudWatchLogsClient(config); + try { + await cwl.send(new DeleteLogGroupCommand({ logGroupName })); + await cwl.send(new CreateLogGroupCommand({ logGroupName })); + } catch (e) { + const res = await cwl.send(new DescribeLogStreamsCommand({ logGroupName, limit: 50 })); + for (const stream of res.logStreams ?? []) { + if (stream.logStreamName) { + await cwl.send(new DeleteLogStreamCommand({ logGroupName, logStreamName: stream.logStreamName })); + } + } + } +} diff --git a/src/lib/ministack/client.ts b/src/lib/ministack/client.ts index d76c817..3b7b6f0 100644 --- a/src/lib/ministack/client.ts +++ b/src/lib/ministack/client.ts @@ -12,6 +12,7 @@ import { SSMClient } from "@aws-sdk/client-ssm"; import { KinesisClient } from "@aws-sdk/client-kinesis"; import { KMSClient } from "@aws-sdk/client-kms"; import { GlueClient } from "@aws-sdk/client-glue"; +import { AthenaClient } from "@aws-sdk/client-athena"; import type { MiniStackConfig } from "./types"; /** @@ -81,3 +82,6 @@ export const getKMSClient = (config: MiniStackConfig) => export const getGlueClient = (config: MiniStackConfig) => new GlueClient(baseConfig(config)); + +export const getAthenaClient = (config: MiniStackConfig) => + new AthenaClient(baseConfig(config));