From f2870f0989af3e45ee06696c20e660c09bd09f6b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 8 Aug 2026 08:20:54 +0200 Subject: [PATCH 1/3] Persist commitment costs on scheduling job meta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cost breakdown was written to rq_job.meta after the job's last save_meta() call, and RQ persists a finishing job with include_meta=False, so the costs were computed and then silently lost from Redis — for every scheduling job, since the feature's introduction in #1946. Any consumer reading job meta (e.g. the jobs API) saw a scheduler_info without them. Save the meta right where the costs are recorded. The regression test fetches a fresh Job instance from Redis, so it asserts what was actually persisted rather than the worker's in-memory object; it was additionally mutation-tested (5/5 targeted mutants killed: fix reverted, save-before-set, wrong key, empty payload, branch inverted). No changelog entry: the feature this repairs is itself unreleased (v1.0.0). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01UKx19DmqHiTBS7iEeFTijs Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 4 ++ .../data/tests/test_scheduling_jobs.py | 50 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 188ea959c2..d169924255 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -845,6 +845,10 @@ def make_schedule( # noqa: C901 continue if rq_job and result.get("name") == "commitment_costs": rq_job.meta["scheduler_info"]["commitment_costs"] = result["data"] + # Persist right away: this runs after the job's last save_meta() call, + # and RQ saves a finishing job with include_meta=False, + # so without an explicit save here the costs never reach Redis. + rq_job.save_meta() continue if "sensor" not in result: continue diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index cb479c6f39..3a9972ab9d 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -100,6 +100,56 @@ def test_scheduling_a_battery( assert out.count(f"Job {job.id} made schedule.") == 1, out +def test_commitment_costs_are_persisted_on_job_meta( + fresh_db, + app, + add_battery_assets_fresh_db, + setup_fresh_test_data, + add_market_prices_fresh_db, +): + """The commitment cost breakdown must survive into the job meta stored in Redis. + + Regression test: the costs used to be written to ``rq_job.meta`` after the job's last + ``save_meta()`` call, and RQ persists a finishing job with ``include_meta=False``, + so they were computed and then silently lost — for every scheduling job. + Fetching a fresh Job instance (rather than inspecting the worker's in-memory object) + is what makes this test see only what actually reached Redis. + """ + battery = next( + s + for s in add_battery_assets_fresh_db["Test battery"].sensors + if s.name == "power" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 2)) + end = tz.localize(datetime(2015, 1, 3)) + + job = create_scheduling_job( + asset_or_sensor=battery, + start=start, + end=end, + belief_time=start, + resolution=timedelta(minutes=15), + flex_model={ + "roundtrip-efficiency": "98%", + "storage-efficiency": 0.999, + }, + ) + work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) + + # Fetch fresh from Redis: in-memory meta mutations on the worker's job object don't count. + finished_job = Job.fetch(job.id, connection=app.queues["scheduling"].connection) + assert finished_job.is_finished + commitment_costs = finished_job.meta["scheduler_info"]["commitment_costs"] + assert ( + commitment_costs + ), "the scheduler reported commitment costs, so the persisted job meta must carry them" + assert all( + isinstance(cost, float) and np.isfinite(cost) + for cost in commitment_costs.values() + ) + + scheduler_specs = { "module": None, # use make_module_descr, see below "class": "DummyScheduler", From 3b2e7be5a585546a3a35be6db2814279200350ff Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 8 Aug 2026 08:24:41 +0200 Subject: [PATCH 2/3] Address review: break docstring lines only after punctuation Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01UKx19DmqHiTBS7iEeFTijs Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_scheduling_jobs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index 3a9972ab9d..e0e2ddaa6f 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -109,11 +109,11 @@ def test_commitment_costs_are_persisted_on_job_meta( ): """The commitment cost breakdown must survive into the job meta stored in Redis. - Regression test: the costs used to be written to ``rq_job.meta`` after the job's last - ``save_meta()`` call, and RQ persists a finishing job with ``include_meta=False``, + Regression test: the costs used to be written to ``rq_job.meta`` after the job's last ``save_meta()`` call, + and RQ persists a finishing job with ``include_meta=False``, so they were computed and then silently lost — for every scheduling job. - Fetching a fresh Job instance (rather than inspecting the worker's in-memory object) - is what makes this test see only what actually reached Redis. + Fetching a fresh Job instance is what makes this test see only what actually reached Redis, + rather than the worker's in-memory job object. """ battery = next( s From 811b2362aa6552ac7854528d63a251eee3a14c1f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 8 Aug 2026 13:16:13 +0200 Subject: [PATCH 3/3] data/tests: assert the persisted commitment costs in the existing battery test Context: - Review: a whole new scheduling test is not needed just to assert these; the checks belong in an existing scheduling test Change: - Moved the assertions into test_scheduling_a_battery, which already runs the same job with the same fixtures, and dropped the separate test - Still fetches a fresh Job from Redis, so only what was actually persisted is asserted (verified to fail when the fix is reverted) Signed-off-by: F.N. Claessen --- .../data/tests/test_scheduling_jobs.py | 54 ++++--------------- 1 file changed, 10 insertions(+), 44 deletions(-) diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index e0e2ddaa6f..2842a066ae 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -37,6 +37,7 @@ def test_scheduling_a_battery( """Test one clean run of one scheduling job: - data source was made, - schedule has been made + - the commitment costs reached the job meta stored in Redis — regression for #2418 - success is logged once (not before compute) — regression for #2049 """ @@ -94,50 +95,10 @@ def test_scheduling_a_battery( sum(v.event_value for v in power_values) < -0.5 ), "some cycling should have occurred to make a profit, resulting in overall consumption due to losses" - # Regression #2049: success message only after compute, not the pre-compute copy-paste echo. - # Count only this job's message — the RQ worker may also process other queued jobs. - out = capsys.readouterr().out - assert out.count(f"Job {job.id} made schedule.") == 1, out - - -def test_commitment_costs_are_persisted_on_job_meta( - fresh_db, - app, - add_battery_assets_fresh_db, - setup_fresh_test_data, - add_market_prices_fresh_db, -): - """The commitment cost breakdown must survive into the job meta stored in Redis. - - Regression test: the costs used to be written to ``rq_job.meta`` after the job's last ``save_meta()`` call, - and RQ persists a finishing job with ``include_meta=False``, - so they were computed and then silently lost — for every scheduling job. - Fetching a fresh Job instance is what makes this test see only what actually reached Redis, - rather than the worker's in-memory job object. - """ - battery = next( - s - for s in add_battery_assets_fresh_db["Test battery"].sensors - if s.name == "power" - ) - tz = pytz.timezone("Europe/Amsterdam") - start = tz.localize(datetime(2015, 1, 2)) - end = tz.localize(datetime(2015, 1, 3)) - - job = create_scheduling_job( - asset_or_sensor=battery, - start=start, - end=end, - belief_time=start, - resolution=timedelta(minutes=15), - flex_model={ - "roundtrip-efficiency": "98%", - "storage-efficiency": 0.999, - }, - ) - work_on_rq(app.queues["scheduling"], exc_handler=exception_reporter) - - # Fetch fresh from Redis: in-memory meta mutations on the worker's job object don't count. + # Regression #2418: the commitment costs used to be written to the job meta after the + # job's last save_meta() call, and RQ persists a finishing job with include_meta=False, + # so they were computed and then silently lost. Fetch a fresh Job to see only what + # actually reached Redis, rather than the worker's in-memory job object. finished_job = Job.fetch(job.id, connection=app.queues["scheduling"].connection) assert finished_job.is_finished commitment_costs = finished_job.meta["scheduler_info"]["commitment_costs"] @@ -149,6 +110,11 @@ def test_commitment_costs_are_persisted_on_job_meta( for cost in commitment_costs.values() ) + # Regression #2049: success message only after compute, not the pre-compute copy-paste echo. + # Count only this job's message — the RQ worker may also process other queued jobs. + out = capsys.readouterr().out + assert out.count(f"Job {job.id} made schedule.") == 1, out + scheduler_specs = { "module": None, # use make_module_descr, see below