Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 96 additions & 3 deletions sustainml_modules/sustainml_modules/sustainml-wp5/backend_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,9 +349,9 @@ def hf_models_compare():

token = os.getenv("HF_TOKEN")
facts = [_model_facts(m, token) for m in models]
print(f"[COMPARE] models={[f['model_id'] for f in facts]}", flush=True)

prompt = _build_compare_prompt(facts)
print(f"[COMPARE] prompt_len={len(prompt)}", flush=True)

try:
llm_text = _ollama_chat(prompt, model="llama3")
Expand Down Expand Up @@ -447,7 +447,14 @@ def results_args():
node_id = data.get('node_id')
json_task = data.get('task_id')
if json_task is not None:
task_id = sustainml_swig.set_task_id(json_task.get('problem_id', 0), json_task.get('iteration_id', 0))
problem_id = json_task.get('problem_id', 0)
iteration_id = json_task.get('iteration_id', 0)
# problem_id/iteration_id map to uint32_t on the C++ side - a negative id
# (e.g. the frontend's -1 "no real problem yet" placeholder) would otherwise
# crash this whole request with an OverflowError.
if problem_id < 0 or iteration_id < 0:
return jsonify({'error': 'problem_id and iteration_id must be non-negative'}), 400
task_id = sustainml_swig.set_task_id(problem_id, iteration_id)
else:
task_id = None

Expand All @@ -472,6 +479,87 @@ def results_args():
return jsonify({utils.string_node(node_id): orchestrator.get_results(node_id, task_id)}), 200


# List the names of every save file that currently exists, for a Load picker.
@server.route('/saved_files', methods=['GET', 'POST'])
def saved_files():
return jsonify({'names': orchestrator.list_saved_files()}), 200


# Explicitly save the given (currently open/live) tasks into a named file.
@server.route('/save_tasks', methods=['POST'])
def save_tasks():
data = request.json or {}
name = data.get('name')
tasks = data.get('tasks', [])
if not name or not tasks:
return jsonify({'error': 'name and at least one task are required'}), 400
path = orchestrator.save_tasks_to_file(name, tasks)
return jsonify({'message': f'Saved {len(tasks)} task(s) to "{name}".', 'path': path}), 200


# Load every task from a named save file. Each task is assigned a fresh problem_id
# (returned to the caller) so loaded tasks always appear as new tasks.
@server.route('/load_tasks', methods=['POST'])
def load_tasks():
data = request.json or {}
name = data.get('name')
if not name:
return jsonify({'error': 'name is required'}), 400
return jsonify({'tasks': orchestrator.load_tasks_from_file(name)}), 200


# Explicitly save the given tasks AND the given HF search/comparison history into
# one named file in a single call - the "save everything" flow, as opposed to
# /save_tasks which only ever touches the tasks/problems tables.
@server.route('/save_all', methods=['POST'])
def save_all():
data = request.json or {}
name = data.get('name')
if not name:
return jsonify({'error': 'name is required'}), 400
tasks = data.get('tasks', [])
hf_searches = data.get('hf_searches', [])
hf_comparisons = data.get('hf_comparisons', [])
orchestrator.save_tasks_to_file(name, tasks)
path = orchestrator.save_hf_state(name, hf_searches, hf_comparisons)
return jsonify({'message': f'Saved everything to "{name}".', 'path': path}), 200


# Load every task AND the HF search/comparison history from a named save file.
@server.route('/load_all', methods=['POST'])
def load_all():
data = request.json or {}
name = data.get('name')
if not name:
return jsonify({'error': 'name is required'}), 400
tasks = orchestrator.load_tasks_from_file(name)
hf_state = orchestrator.load_hf_state(name)
return jsonify({
'tasks': tasks,
'hf_searches': hf_state.get('searches', []),
'hf_comparisons': hf_state.get('comparisons', []),
}), 200


# Delete a single named save file. Does not affect any other save or tasks
# currently in progress.
@server.route('/delete_saved_file', methods=['POST'])
def delete_saved_file():
data = request.json or {}
name = data.get('name')
if not name:
return jsonify({'error': 'name is required'}), 400
orchestrator.delete_saved_file(name)
return jsonify({'message': f'Deleted "{name}".'}), 200


# Delete every save file. Does not affect tasks currently in progress.
@server.route('/wipe_db', methods=['POST'])
def wipe_db():
orchestrator.delete_all_saved_files()
return jsonify({'message': 'All save files deleted.'}), 200


# Flask server shutdown route
@server.route('/shutdown', methods=['GET'])
def shutdown():
Expand All @@ -495,7 +583,12 @@ def __init__(self):
self.orchestrator_thread = threading.Thread(target=orchestrator.run)
# Create Flask server
threading.Thread.__init__(self)
self.srv = make_server(server_ip_address, server_port, server)
# threaded=True: without it, Werkzeug serves one request at a time -
# a single slow /config_request (e.g. dataset_path, which runs a
# multi-minute LLM-based analysis pipeline synchronously) then blocks
# every other concurrent request (status polling, other config
# requests) on this same server for its entire duration.
self.srv = make_server(server_ip_address, server_port, server, threaded=True)
self.ctx = server.app_context()
self.ctx.push()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""SustainML Orchestrator Node API specification."""

from . import utils
from . import persistence
import numpy as np

from sustainml_swig import OrchestratorNodeHandle as cpp_OrchestratorNodeHandle
Expand Down Expand Up @@ -117,6 +118,12 @@ def __init__(self):
self._txn_lock = threading.Lock()
self._txn_counter = 0

# Results for tasks loaded from a save file, keyed by string_task(new_task_id) ->
# {node_id: json_dict}. These task_ids are freshly minted on load (see
# load_tasks_from_file), never reused from the file, so they can never collide
# with a task_id created normally in this session, in either direction.
self._loaded_task_cache = {}

# Proxy method to run the node
def run(self):

Expand All @@ -130,6 +137,105 @@ def terminate(self):
def get_last_task_id(self):
return self.handler_.last_task_id

def list_saved_files(self):
return persistence.list_saved_files()

def delete_saved_file(self, name):
persistence.delete_saved_file(name)

def delete_all_saved_files(self):
persistence.delete_all_saved_files()

def save_tasks_to_file(self, name, tasks):
"""Save the given tasks (with their currently-known results) into a named file.

tasks: list of dicts {"problem_id", "iteration_id", "display_name"}. Works for
both live tasks (checked via result_status, skipping any node still pending -
never waited on) and tasks previously loaded from a file (checked via
_loaded_task_cache), so re-saving something you just loaded carries its data
forward correctly instead of picking up an empty live placeholder.

Returns the absolute path of the file that was written.
"""
result_nodes = (
utils.node_id.ORCHESTRATOR,
utils.node_id.APP_REQUIREMENTS,
utils.node_id.CARBONTRACKER,
utils.node_id.HW_CONSTRAINTS,
utils.node_id.HW_PROVIDER,
utils.node_id.ML_MODEL_METADATA,
utils.node_id.ML_MODEL_PROVIDER,
)
payload = []
for task in tasks:
task_id = sustainml_swig.set_task_id(task["problem_id"], task["iteration_id"])
key = utils.string_task(task_id)
status = self.handler_.result_status.get(key, {})
cached = self._loaded_task_cache.get(key, {})

results = {}
for node in result_nodes:
is_live_user_input = node == utils.node_id.ORCHESTRATOR and key in self.handler_.result_status
if not (is_live_user_input or status.get(node.value, False) or node.value in cached):
continue
try:
results[node.value] = self.get_results(node.value, task_id)
except Exception as e:
print(f"[persistence] failed to read {utils.string_node(node.value)} result for task "
f"{utils.string_task(task_id)}: {e}")

payload.append({
"problem_id": task["problem_id"],
"iteration_id": task["iteration_id"],
"display_name": task.get("display_name"),
"results": results,
})

return persistence.save_tasks_to_file(name, payload)

def load_tasks_from_file(self, name):
"""Load every task from the named save file, assigning each a fresh problem_id
so loaded tasks always appear as new tasks and can never collide with anything
already live or created afterward in this session.

Returns a list of dicts {"problem_id" (new), "iteration_id", "display_name"}.
"""
loaded = []
for task in persistence.load_tasks_from_file(name):
# Only the problem_id needs to be fresh to avoid collisions; keep the
# original iteration_id for display continuity.
new_task_id, _ = self.node_.prepare_new_task()
new_problem_id = new_task_id.problem_id()
iteration_id = task["iteration_id"]
display_task_id = sustainml_swig.set_task_id(new_problem_id, iteration_id)
key = utils.string_task(display_task_id)

# Each saved node result still carries the *original* problem_id/iteration_id
# embedded in its own "task_id" field (from when it was first computed live).
# The frontend reads that embedded value - not the id it requested - to decide
# which tab a result belongs to, so it must be rewritten to the freshly
# assigned id here, or a loaded result silently reports as its old task
# instead of the new one.
rewritten_results = {}
for node_id, result in task["results"].items():
result = dict(result)
result["task_id"] = {"problem_id": new_problem_id, "iteration_id": iteration_id}
rewritten_results[node_id] = result
self._loaded_task_cache[key] = rewritten_results

loaded.append({
"problem_id": new_problem_id,
"iteration_id": iteration_id,
"display_name": task.get("display_name"),
})
return loaded

def save_hf_state(self, name, hf_searches, hf_comparisons):
return persistence.save_hf_state(name, hf_searches, hf_comparisons)

def load_hf_state(self, name):
return persistence.load_hf_state(name)

def get_all_status(self):
json_output = {}
for key, value in self.handler_.node_status_.items():
Expand Down Expand Up @@ -326,6 +432,19 @@ def get_results(self, node_id, task_id):
if task_id is None:
task_id = self.get_last_task_id()

key = utils.string_task(task_id)
if key in self._loaded_task_cache:
# A task loaded from a save file never runs through the live pipeline, so a
# node missing from its cache (it was only partially complete when saved)
# will NEVER become available - return immediately instead of falling
# through to the live blocking wait below, which would hang forever and
# stall the single-threaded REST server for every other request too.
cached = self._loaded_task_cache[key]
if node_id in cached:
return cached[node_id]
message = utils.string_node(node_id) + " was not saved for this loaded task."
return {'message': message, 'task_id': utils.task_json(task_id)}

if node_id == utils.node_id.APP_REQUIREMENTS.value:
return self.get_app_requirements_task_data(task_id)
elif node_id == utils.node_id.ML_MODEL_METADATA.value:
Expand Down
Loading
Loading