v5.2.0: DD tool improvements and dump-based graph distribution

.github/workflows/docker-build-push.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Resolve graph tag
         id: graph-tag
         run: |
-          IMAS_PACKAGE="imas-codex-graph-dd"
+          IMAS_PACKAGE="imas-codex-graph"
           OWNER_REGISTRY=$(echo "${{ env.GRAPH_REGISTRY }}" | tr '[:upper:]' '[:lower:]')
           FALLBACK_REGISTRY="${{ env.GRAPH_REGISTRY_FALLBACK }}"
 
@@ -78,8 +78,9 @@ jobs:
             local REGISTRY="$1"
             local TAGS
             TAGS=$(oras repo tags "${REGISTRY}/${IMAS_PACKAGE}" 2>&1) || true
+            # Match only 'latest' or semver tags (v1.2.3, 5.2.0rc8, etc.)
             local CLEAN
-            CLEAN=$(echo "$TAGS" | grep -E '^[0-9v]' || true)
+            CLEAN=$(echo "$TAGS" | grep -E '^(latest|v?[0-9]+\.[0-9]+\.)' || true)
             if echo "$CLEAN" | grep -qx "latest"; then
               echo "latest"
             elif [ -n "$CLEAN" ]; then
@@ -116,7 +117,7 @@ jobs:
         if: steps.graph-tag.outputs.imas-tag != 'none'
         run: |
           REGISTRY="${{ steps.graph-tag.outputs.graph-registry }}"
-          ARTIFACT="${REGISTRY}/imas-codex-graph-dd:${{ steps.graph-tag.outputs.imas-tag }}"
+          ARTIFACT="${REGISTRY}/imas-codex-graph:${{ steps.graph-tag.outputs.imas-tag }}"
           echo "Pulling: ${ARTIFACT}"
           mkdir -p /tmp/graph-dump
           oras pull "${ARTIFACT}" -o /tmp/graph-dump --allow-path-traversal
@@ -134,7 +135,7 @@ jobs:
           docker stop $(docker ps -q --filter "ancestor=neo4j:2026.01.4-community") || true
           sleep 3
 
-      - name: Load graph dump into Neo4j
+      - name: Load graph data into Neo4j
         if: steps.graph-tag.outputs.imas-tag != 'none'
         run: |
           set -euo pipefail
@@ -145,7 +146,9 @@ jobs:
           mkdir -p /tmp/graph-extracted
           tar -xzf "${ARCHIVE}" -C /tmp/graph-extracted
 
-          # Find the dump file (may be named graph.dump or neo4j.dump)
+          NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1)
+          NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}')
+
           DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1)
           if [ -z "${DUMP_FILE}" ]; then
             echo "ERROR: No .dump file found in archive"
@@ -154,17 +157,11 @@ jobs:
           fi
           echo "Found dump: ${DUMP_FILE}"
 
-          # neo4j-admin expects the dump file named neo4j.dump
           DUMP_DIR=$(dirname "${DUMP_FILE}")
           if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then
             cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump"
           fi
 
-          # Get the Neo4j container ID for data volume
-          NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1)
-          NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}')
-
-          # Load into Neo4j using a fresh container
           docker run --rm \
             -v "${NEO4J_DATA_VOLUME}:/data" \
             -v "${DUMP_DIR}:/dump" \
@@ -221,7 +218,7 @@ jobs:
 
       - name: Install dependencies
         if: steps.graph-tag.outputs.imas-tag != 'none'
-        run: uv sync --extra test
+        run: uv sync --extra test --no-dev
         env:
           HATCH_BUILD_NO_HOOKS: true
 
@@ -284,8 +281,10 @@ jobs:
         run: |
           echo "Before cleanup:"
           df -h /
-          # Remove packages we don't need for docker builds
-          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          # Aggressively free space — Docker build with Neo4j graph needs ~25 GB
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+            /opt/hostedtoolcache /usr/share/swift /usr/local/share/boost \
+            /usr/local/graalvm /usr/local/share/chromium /usr/local/lib/node_modules
           sudo docker image prune --all --force
           echo "After cleanup:"
           df -h /

.github/workflows/graph-quality.yml

@@ -89,12 +89,18 @@ jobs:
         run: |
           echo "${{ secrets.GHCR_TOKEN }}" | oras login ghcr.io -u token --password-stdin
 
-      - name: Pull graph dump from GHCR
+      - name: Pull graph from GHCR
         run: |
           ARTIFACT="${{ steps.resolve-tag.outputs.registry }}/imas-codex-graph:${{ steps.resolve-tag.outputs.tag }}"
           echo "Pulling: ${ARTIFACT}"
           mkdir -p /tmp/graph-dump
-          oras pull "${ARTIFACT}" -o /tmp/graph-dump
+          oras pull "${ARTIFACT}" -o /tmp/graph-dump --allow-path-traversal
+          # Handle oras path traversal: artifact may land outside -o dir
+          FOUND=$(find /tmp -maxdepth 3 -name "*.tar.gz" ! -path "/tmp/graph-dump/*" 2>/dev/null | head -1)
+          if [ -n "$FOUND" ] && [ ! -f /tmp/graph-dump/*.tar.gz ]; then
+            echo "Moving artifact from traversal path: ${FOUND}"
+            mv "$FOUND" /tmp/graph-dump/
+          fi
           ls -la /tmp/graph-dump/
 
       - name: Wait for Neo4j to be ready
@@ -108,13 +114,12 @@ jobs:
             sleep 2
           done
 
-      - name: Stop Neo4j for dump load
+      - name: Stop Neo4j for data load
         run: |
-          # Stop Neo4j service container to load dump
           docker stop $(docker ps -q --filter "ancestor=neo4j:2026.01.4-community") || true
           sleep 3
 
-      - name: Load graph dump into Neo4j
+      - name: Load graph data into Neo4j
         run: |
           set -euo pipefail
           ARCHIVE=$(ls /tmp/graph-dump/*.tar.gz | head -1)
@@ -124,40 +129,36 @@ jobs:
           mkdir -p /tmp/graph-extracted
           tar -xzf "${ARCHIVE}" -C /tmp/graph-extracted
 
-          # Find the graph.dump file
-          DUMP_FILE=$(find /tmp/graph-extracted -name "graph.dump" | head -1)
+          NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1)
+          NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}')
+
+          # Dump-based archive
+          DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1)
           if [ -z "${DUMP_FILE}" ]; then
-            echo "ERROR: No graph.dump found in archive"
+            echo "ERROR: No .dump file found in archive"
             ls -laR /tmp/graph-extracted/
             exit 1
           fi
           echo "Found dump: ${DUMP_FILE}"
 
-          # neo4j-admin load expects the file named <database>.dump
-          DUMP_DIR=$(mktemp -d)
-          cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump"
-          chmod -R 777 "${DUMP_DIR}"
-
-          # Get the Neo4j container ID for data volume
-          NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1)
-          NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}')
+          DUMP_DIR=$(dirname "${DUMP_FILE}")
+          if [ "$(basename ${DUMP_FILE})" != "neo4j.dump" ]; then
+            cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump"
+          fi
 
-          # Load into Neo4j using a fresh container
-          docker run --rm --user root \
+          docker run --rm \
             -v "${NEO4J_DATA_VOLUME}:/data" \
             -v "${DUMP_DIR}:/dump" \
             neo4j:2026.01.4-community \
             neo4j-admin database load neo4j \
               --from-path=/dump \
-              --overwrite-destination=true \
-              --verbose
+              --overwrite-destination=true
 
       - name: Start Neo4j with loaded data
         run: |
           NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1)
           docker start "${NEO4J_CONTAINER}"
 
-          # Wait for Neo4j to come back up
           echo "Waiting for Neo4j to restart..."
           for i in $(seq 1 60); do
             if curl -sf http://localhost:7474/ > /dev/null 2>&1; then
@@ -174,13 +175,11 @@ jobs:
 
       - name: Reset Neo4j password
         run: |
-          # After dump load, auth is reset. Set the password.
           NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1)
           docker exec "${NEO4J_CONTAINER}" neo4j-admin dbms set-initial-password imas-codex 2>/dev/null || true
 
       - name: Verify graph is loaded
         run: |
-          # Quick sanity check with cypher-shell
           NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1)
           docker exec "${NEO4J_CONTAINER}" cypher-shell \
             -u neo4j -p imas-codex \
@@ -196,7 +195,7 @@ jobs:
         run: uv python install 3.12
 
       - name: Install dependencies
-        run: uv sync --extra test
+        run: uv sync --extra test --no-dev
         env:
           HATCH_BUILD_NO_HOOKS: true
 

.github/workflows/test.yml

@@ -60,7 +60,7 @@ jobs:
         run: uv python install ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: uv sync --extra test
+        run: uv sync --extra test --no-dev
         env:
           HATCH_BUILD_NO_HOOKS: true
           IMAS_DD_VERSION: ${{ matrix.imas-dd-version }}
@@ -128,7 +128,7 @@ jobs:
         run: uv python install 3.12
 
       - name: Install dependencies
-        run: uv sync --extra test
+        run: uv sync --extra test --no-dev
         env:
           HATCH_BUILD_NO_HOOKS: true
 

Dockerfile

@@ -153,8 +153,8 @@ RUN --mount=type=secret,id=GHCR_TOKEN \
         touch /tmp/graph-pull/.no-graph; \
     fi
 
-## Stage 4: Load graph dump into Neo4j data directory
-# Uses neo4j-admin from the Neo4j image to load the dump
+## Stage 4: Load graph data into Neo4j data directory
+# Graph archives contain a neo4j.dump file inside a tar.gz.
 FROM neo4j:2026.01.4-community AS graph-loader
 
 # Propagate GRAPH_TAG to bust cache when graph version changes
@@ -164,89 +164,56 @@ RUN echo "Graph tag: ${GRAPH_TAG}" > /dev/null
 # Copy graph archive from builder
 COPY --from=builder /tmp/graph-pull/ /tmp/graph-pull/
 
-# Extract and load the graph dump (or create empty database)
-# Handles both raw .dump files (oras pull) and .tar.gz archives.
+# Extract and load graph data into Neo4j data directory.
 # CRITICAL: clean up intermediate files progressively to minimize peak disk usage.
-# The graph dump is ~5 GB; without cleanup we'd have archive + extracted + copy + loaded
-# data all on disk simultaneously (~15+ GB), exceeding CI runner capacity.
+# NOTE: Neo4j pre-start recovery is deferred to container startup (entrypoint.sh)
+# to avoid ~2.3 GB WAL creation that exceeds CI runner disk during Docker build.
 RUN set -ex && \
     if [ -f /tmp/graph-pull/.no-graph ]; then \
-        echo "⚠ No graph data — creating empty Neo4j database"; \
+        echo "No graph data — creating empty Neo4j database"; \
         mkdir -p /data/databases/neo4j /data/transactions/neo4j; \
     else \
         cd /tmp/graph-pull && \
         DUMP=$(ls *.dump 2>/dev/null | head -1) && \
         ARCHIVE=$(ls *.tar.gz 2>/dev/null | head -1) && \
-        mkdir -p /tmp/dumps && \
         if [ -n "$DUMP" ]; then \
             echo "Loading dump directly: $DUMP" && \
+            mkdir -p /tmp/dumps && \
             mv "$DUMP" /tmp/dumps/neo4j.dump && \
-            rm -rf /tmp/graph-pull; \
+            rm -rf /tmp/graph-pull && \
+            cd / && \
+            neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination=true 2>&1 && \
+            rm -rf /tmp/dumps && \
+            echo "Graph loaded from dump"; \
         elif [ -n "$ARCHIVE" ]; then \
             echo "Extracting: $ARCHIVE" && \
             mkdir -p /tmp/graph-extracted && \
             tar -xzf "$ARCHIVE" -C /tmp/graph-extracted && \
             rm -rf /tmp/graph-pull && \
-            DUMP=$(find /tmp/graph-extracted -name "*.dump" -type f | head -1) && \
-            if [ -z "$DUMP" ]; then \
-                echo "ERROR: No .dump file found in archive" >&2; \
+            DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" -type f | head -1) && \
+            if [ -z "$DUMP_FILE" ]; then \
+                echo "ERROR: No .dump found in archive" >&2; \
                 find /tmp/graph-extracted -type f >&2; \
                 exit 1; \
             fi && \
-            echo "Found dump: $DUMP ($(du -sh "$DUMP" | cut -f1))" && \
-            mv "$DUMP" /tmp/dumps/neo4j.dump && \
-            rm -rf /tmp/graph-extracted; \
+            echo "Found dump: $DUMP_FILE ($(du -sh "$DUMP_FILE" | cut -f1))" && \
+            mkdir -p /tmp/dumps && \
+            mv "$DUMP_FILE" /tmp/dumps/neo4j.dump && \
+            rm -rf /tmp/graph-extracted && \
+            cd / && \
+            neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination=true 2>&1 && \
+            rm -rf /tmp/dumps && \
+            echo "Graph loaded from dump"; \
         else \
             echo "ERROR: No .dump or .tar.gz found in /tmp/graph-pull/" >&2; \
             ls -la /tmp/graph-pull/ >&2; \
             exit 1; \
-        fi && \
-        echo "Loading dump into Neo4j ($(du -sh /tmp/dumps/neo4j.dump | cut -f1))..." && \
-        df -h / && \
-        cd / && \
-        neo4j-admin database load neo4j --from-path=/tmp/dumps --overwrite-destination 2>&1 && \
-        rm -rf /tmp/dumps && \
-        echo "✓ Graph loaded into Neo4j data directory"; \
+        fi; \
     fi
 
-# Pre-start Neo4j to complete WAL recovery and create system DB.
-# This shifts the expensive recovery from runtime (slow Azure I/O)
-# to build time (fast CI SSD). The database ships fully recovered.
-RUN if [ ! -f /tmp/graph-pull/.no-graph ]; then \
-    echo "Pre-starting Neo4j for database recovery..." && \
-    echo "dbms.security.auth_enabled=false" >> /var/lib/neo4j/conf/neo4j.conf && \
-    /var/lib/neo4j/bin/neo4j console > /tmp/neo4j-recovery.log 2>&1 & \
-    NEO4J_PID=$! && \
-    READY=0 && \
-    for i in $(seq 1 120); do \
-        if /var/lib/neo4j/bin/cypher-shell -a bolt://127.0.0.1:7687 "RETURN 1" > /dev/null 2>&1; then \
-            echo "✓ Database recovered (${i}s)"; \
-            READY=1; \
-            break; \
-        fi; \
-        if ! kill -0 $NEO4J_PID 2>/dev/null; then \
-            echo "ERROR: Neo4j exited during recovery. Log:"; \
-            cat /tmp/neo4j-recovery.log; \
-            exit 1; \
-        fi; \
-        sleep 1; \
-    done && \
-    if [ "$READY" -eq 0 ]; then \
-        echo "ERROR: Recovery did not complete in 120s. Log:"; \
-        tail -50 /tmp/neo4j-recovery.log; \
-        exit 1; \
-    fi && \
-    /var/lib/neo4j/bin/neo4j stop && \
-    sleep 2 && \
-    rm -f /tmp/neo4j-recovery.log && \
-    echo "✓ Neo4j shut down cleanly — database is recovery-free"; \
-fi
-
-# NOTE: Do NOT remove transaction logs (/data/transactions/neo4j/*).
-# Neo4j 2026 requires valid WAL state to open the database after
-# neo4j-admin load.  Deleting tx logs causes Neo4j HTTP to start but
-# the bolt database remains offline — all Cypher queries fail silently.
-# The ~2.3 GB cost is acceptable for a working container.
+# NOTE: Neo4j recovery happens at container startup (entrypoint.sh waits up to
+# 180s for database readiness). This adds ~10-30s to first startup but avoids
+# the ~2.3 GB WAL creation that would exceed CI runner disk during Docker build.
 
 ## Stage 5: Final runtime image (assemble from builder + Neo4j + graph data)
 FROM python:3.12-slim