Resumable HTTP uploads

.github/dependabot.yml

@@ -7,9 +7,4 @@ updates:
     # Check for updates once a week
     schedule:
       interval: "weekly"
-    # Group actions version bumps into a single PR
-    groups:
-      actions-deps:
-        patterns:
-          - "*"
 

alembic/versions/b2c52ee8ff12_add_ingestion_status.py

@@ -0,0 +1,71 @@
+"""Add ingestion status
+
+Revision ID: b2c52ee8ff12
+Revises: 9e9a4a7cd639
+Create Date: 2026-05-11 16:16:03.768893
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b2c52ee8ff12"
+down_revision: Union[str, Sequence[str], None] = "28bee3aa2429"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    conn = op.get_bind()
+    dialect = conn.dialect.name
+    if dialect == "postgresql":
+        op.execute(
+            "CREATE TYPE ingestionstatus AS ENUM ('QUEUED', 'COPYING', 'COPIED', "
+            "'VALIDATING', 'VALIDATED', 'COMPLETED', 'COPY_FAILED', "
+            "'VALIDATION_FAILED')"
+        )
+    with op.batch_alter_table("simulations", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "ingestion_status",
+                sa.Enum(
+                    "QUEUED",
+                    "COPYING",
+                    "COPIED",
+                    "VALIDATING",
+                    "VALIDATED",
+                    "COMPLETED",
+                    "COPY_FAILED",
+                    "VALIDATION_FAILED",
+                    name="ingestionstatus",
+                ),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(sa.Column("ingestion_version", sa.Integer(), nullable=True))
+    op.execute(
+        "UPDATE simulations SET ingestion_status = 'COMPLETED' WHERE ingestion_status "
+        "IS NULL"
+    )
+    op.execute(
+        "UPDATE simulations SET ingestion_version = 0 WHERE ingestion_version IS NULL"
+    )
+    with op.batch_alter_table("simulations", schema=None) as batch_op:
+        batch_op.alter_column("ingestion_status", nullable=False)
+        batch_op.alter_column("ingestion_version", nullable=False)
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    with op.batch_alter_table("simulations", schema=None) as batch_op:
+        batch_op.drop_column("ingestion_version")
+        batch_op.drop_column("ingestion_status")
+    conn = op.get_bind()
+    dialect = conn.dialect.name
+    if dialect == "postgresql":
+        op.execute("DROP TYPE ingestionstatus")

config/simdb.cfg

@@ -34,3 +34,4 @@ result_backend = redis://redis:6379/0
 
 [partition]
 data = /data/simdb/partition
+http = /data/simdb/http

docker-compose.yml

@@ -20,6 +20,7 @@ services:
       - ./validation:/app/validation:ro
       - ./config:/app/config:ro
       - ./tmp/partition_data:/data/simdb/partition:ro
+      - ./tmp/http:/data/simdb/http
       - ./upload_folder:/data/simdb/simulations
     depends_on:
       redis:
@@ -38,6 +39,7 @@ services:
     volumes:
       - ./config:/app/config:ro
       - ./tmp/partition_data:/data/simdb/partition:ro
+      - ./tmp/http:/data/simdb/http
       - ./upload_folder:/data/simdb/simulations
     depends_on:
       redis:

docs/Makefile

@@ -11,13 +11,7 @@ BUILDDIR      = _build
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: help html Makefile
-
-# Copy source files from docs/ into sphinx/ before building (mirrors .readthedocs.yml)
-html: Makefile
-	cp ../*.md $(SOURCEDIR)/ 2>/dev/null || true
-	cp ../*.svg $(SOURCEDIR)/ 2>/dev/null || true
-	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+.PHONY: help Makefile
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).

docs/celery.md

@@ -0,0 +1,77 @@
+# Celery async task processing
+
+SimDB uses [Celery](https://docs.celeryproject.org/) to run asynchronous background
+tasks such as copying simulation files and completing the ingestion pipeline.
+
+## Overview
+
+When simulations are uploaded via the REST API, the server offloads heavy operations
+to Celery workers instead of blocking the HTTP request. Tasks are defined in
+`src/simdb/workers/tasks.py`:
+
+- `copy_files_task` — copies input/output files from source locations to the server's
+  upload folder and updates the simulation's ingestion status.
+- `complete_ingestion_task` — marks a simulation as fully ingested.
+- `validate_imas_task` — runs validation checks on IMAS data (placeholder).
+- `send_email_task` — sends email notifications.
+
+Tasks can be chained in the API endpoint:
+
+```python
+copy_files = copy_files_task.si(simulation.uuid, ...)
+complete = complete_ingestion_task.si(simulation.uuid)
+_ = (copy_files | complete).apply_async()
+```
+
+## Configuration
+
+Celery is configured via `app.cfg`:
+
+| Section | Option         | Required | Description                                      |
+|---------|----------------|----------|--------------------------------------------------|
+| celery  | broker_url     | no       | Redis URL for the message broker. Defaults to `redis://localhost:6379/0` |
+| celery  | result_backend | no       | Redis URL for results storage. Defaults to `redis://localhost:6379/0`   |
+
+Example:
+
+```ini
+[celery]
+broker_url = redis://localhost:6379/0
+result_backend = redis://localhost:6379/0
+```
+
+## Running workers
+
+### Standalone worker
+
+Start a Celery worker using the built-in CLI:
+
+```bash
+simdb_worker
+```
+
+### Worker with beat scheduler
+
+For periodic tasks (e.g. cleanup, reports), run both the worker and beat:
+
+```bash
+# Terminal 1: worker
+simdb_worker
+
+# Terminal 2: beat scheduler
+simdb_beat
+```
+
+### Flower monitoring
+
+[Flower](https://flower.readthedocs.io/) provides a web UI for monitoring Celery
+workers and tasks:
+
+```bash
+celery -A simdb.workers.celery flower --port=5555
+```
+
+## Testing with eager mode
+
+In tests, set `task_always_eager = True` to run tasks synchronously without a
+broker.

docs/cli.md

@@ -14,7 +14,6 @@ Options:
 Commands:
   alias       Query remote and local aliases.
   config      Query/update application configuration.
-  database    Manage local simulation database.
   manifest    Create/check manifest file.
   provenance  Create the PROVENANCE_FILE from the current system.
   remote      Interact with the remote SimDB service.
@@ -142,7 +141,6 @@ Options:
   --help  Show this message and exit.
 ```
 
-
 ## Manifest
 
 
@@ -421,30 +419,38 @@ Usage: simdb remote [NAME] query [OPTIONS] [CONSTRAINTS]...
       NAME=[mod]VALUE
 
   Where `[mod]` is an optional query modifier. Available query modifiers are:
-      eq:  - This checks for equality (this is the same behaviour as not providing any modifier).
+      eq:  - This checks for equality (this is the same behaviour as not providing
+             any modifier).
       in:  - This searches inside the value instead of looking for exact matches.
       gt:  - This checks for values greater than the given quantity.
       agt: - This checks for any array elements are greater than the given quantity.
       ge:  - This checks for values greater than or equal to the given quantity.
-      age: - This checks for any array elements are greater than or equal to the given quantity.
+      age: - This checks for any array elements are greater than or equal to the given
+             quantity.
       lt:  - This checks for values less than the given quantity.
-      alt:  - This checks for any array elements are less than the given quantity.
+      alt: - This checks for any array elements are less than the given quantity.
       le:  - This checks for values less than or equal to the given quantity.
-      ale:  - This checks for any array elements are less than or equal to the given quantity.
+      ale: - This checks for any array elements are less than or equal to the given
+              quantity.
 
   Modifier examples:
       alias=eq:foo                                                performs exact match
-      summary.code.name=in:foo                                    matches all names containing foo
-      summary.heating_current_drive.power_additional.value=agt:0  matches all simulations where any array element
-      of summary.heating_current_drive.power_additional.value is greater than 0
+      summary.code.name=in:foo                                    matches all names
+          containing foo
+      summary.heating_current_drive.power_additional.value=agt:0  matches all
+          simulations where any array element of
+          summary.heating_current_drive.power_additional.value is greater than 0
 
-  Any string comparisons are done in a case-insensitive manner. If multiple constraints are provided then simulations
-  are returned that match all given constraints.
+  Any string comparisons are done in a case-insensitive manner. If multiple
+  constraints are provided then simulations are returned that match all given
+  constraints.
 
   Examples:
-      sim remote query workflow.name=in:test       finds all simulations where workflow.name contains test
-                                                       (case-insensitive)
-      sim remote query pulse=gt:1000 run=0         finds all simulations where pulse is > 1000 and run = 0
+      sim remote query workflow.name=in:test       finds all simulations where
+                                                   workflow.name contains test
+                                                   (case-insensitive)
+      sim remote query pulse=gt:1000 run=0         finds all simulations where pulse
+                                                   is > 1000 and run = 0
 
 Options:
   -m, --meta-data TEXT  Additional meta-data field to print.
@@ -603,20 +609,22 @@ Options:
   --help  Show this message and exit.
 
 Commands:
-  delete    Delete the ingested simulation with given SIM_ID (UUID or...
-  info      Print information on the simulation with given SIM_ID (UUID...
-  ingest    Ingest a MANIFEST_FILE.
-  list      List ingested simulations.
-  modify    Modify the ingested simulation.
-  pull      Pull the simulation with the given SIM_ID (UUID or alias)...
-  push      Push the simulation with the given SIM_ID (UUID or alias) to...
-  query     Perform a metadata query to find matching local simulations.
-  validate  Validate the ingested simulation with given SIM_ID (UUID or...
+  delete      Delete the ingested simulation with given SIM_ID (UUID or...
+  info        Print information on the simulation with given SIM_ID (UUID...
+  ingest      Ingest a MANIFEST_FILE.
+  list        List ingested simulations.
+  modify      Modify the ingested simulation.
+  pull        Pull the simulation with the given SIM_ID (UUID or alias)...
+  push        Push the simulation with the given SIM_ID (UUID or alias)...
+  push_http   Push the simulation with the given SIM_ID to the REMOTE...
+  push_local  Push the simulation with the given SIM_ID (UUID or alias)...
+  query       Perform a metadata query to find matching local simulations.
+  validate    Validate the ingested simulation with given SIM_ID (UUID or...
 ```
 
 
 ```text
-Usage: simdb simulation delete [OPTIONS] SIM_ID
+Usage: simdb simulation delete [OPTIONS] [SIM_ID]
 
   Delete the ingested simulation with given SIM_ID (UUID or alias).
 
@@ -703,6 +711,39 @@ Options:
 ```
 
 
+```text
+Usage: simdb simulation push_http [OPTIONS] [REMOTE] SIM_ID
+
+  Push the simulation with the given SIM_ID to the REMOTE over resumable HTTP.
+
+  Unlike push_local, this does not require a filesystem shared with the
+  server: the file bytes are uploaded over HTTP using a resumable protocol and
+  staged into the server's 'http' partition. An interrupted push can be
+  resumed by re-running the command.
+
+Options:
+  --username TEXT  Username used to authenticate with the remote.
+  --password TEXT  Password used to authenticate with the remote.
+  --replaces TEXT  SIM_ID of simulation to deprecate and replace.
+  --add-watcher    Add the current user as a watcher of the simulation.
+  --help           Show this message and exit.
+```
+
+
+```text
+Usage: simdb simulation push_local [OPTIONS] [REMOTE] SIM_ID
+
+  Push the simulation with the given SIM_ID (UUID or alias) to the REMOTE.
+
+Options:
+  --username TEXT  Username used to authenticate with the remote.
+  --password TEXT  Password used to authenticate with the remote.
+  --replaces TEXT  SIM_ID of simulation to deprecate and replace.
+  --add-watcher    Add the current user as a watcher of the simulation.
+  --help           Show this message and exit.
+```
+
+
 ```text
 Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]...
 
@@ -712,7 +753,8 @@ Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]...
       NAME=[mod]VALUE
 
   Where `[mod]` is an optional query modifier. Available query modifiers are:
-      eq: - This checks for equality (this is the same behaviour as not providing any modifier).
+      eq: - This checks for equality (this is the same behaviour as not providing any
+            modifier).
       ne: - This checks for value that do not equal.
       in: - This searches inside the value instead of looking for exact matches.
       ni: - This searches inside the value for elements that do not match.
@@ -722,22 +764,26 @@ Usage: simdb simulation query [OPTIONS] [CONSTRAINTS]...
       le: - This checks for values less than or equal to the given quantity.
 
   For the following modifiers, VALUE should not be provided.     exist: - This
-  returns simulations where metadata with NAME exists, regardless of the
-  value.
+  returns simulations where metadata with NAME exists, regardless
+  of the value.
 
   Modifier examples:
       responsible_name=foo        performs exact match
       responsible_name=in:foo     matches all names containing foo
       pulse=gt:1000               matches all pulses > 1000
-      sequence=exist:             matches all simulations that have "sequence" metadata values
+      sequence=exist:             matches all simulations that have "sequence"
+                                  metadata values
 
-  Any string comparisons are done in a case-insensitive manner. If multiple constraints are provided then simulations
-  are returned that match all given constraints.
+  Any string comparisons are done in a case-insensitive manner. If multiple
+  constraints are provided then simulations are returned that match all given
+  constraints.
 
   Examples:
-      sim simulation query workflow.name=in:test       finds all simulations where workflow.name contains test
+      sim simulation query workflow.name=in:test       finds all simulations where
+                                                       workflow.name contains test
                                                        (case-insensitive)
-      sim simulation query pulse=gt:1000 run=0         finds all simulations where pulse is > 1000 and run = 0
+      sim simulation query pulse=gt:1000 run=0         finds all simulations where
+                                                       pulse is > 1000 and run = 0
 
 Options:
   -m, --meta-data TEXT  Additional meta-data field to print.

docs/cli.md.in

@@ -10,10 +10,6 @@
 
 {{ config }}
 
-## Database
-
-{{ database }}
-
 ## Manifest
 
 {{ manifest }}