From 3a4b9222ceef28290431ca0197d31e35d8165fdd Mon Sep 17 00:00:00 2001 From: Stephen Nneji Date: Tue, 12 May 2026 14:47:50 +0100 Subject: [PATCH 1/6] Add function to run directly from MATLAB API --- ratapi/__init__.py | 3 +- ratapi/utils/matlab.py | 67 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 ratapi/utils/matlab.py diff --git a/ratapi/__init__.py b/ratapi/__init__.py index 98683c7e..8f441f44 100644 --- a/ratapi/__init__.py +++ b/ratapi/__init__.py @@ -9,7 +9,7 @@ from ratapi.outputs import BayesResults, Results from ratapi.project import Project from ratapi.run import run -from ratapi.utils import convert, plotting +from ratapi.utils import convert, matlab, plotting with suppress(ImportError): # orsopy is an optional dependency from ratapi.utils import orso as orso @@ -26,4 +26,5 @@ "run", "plotting", "convert", + "matlab", ] diff --git a/ratapi/utils/matlab.py b/ratapi/utils/matlab.py new file mode 100644 index 00000000..dad8921b --- /dev/null +++ b/ratapi/utils/matlab.py @@ -0,0 +1,67 @@ +"""Runs RAT from the MATLAB API.""" + +import os +import tempfile +import warnings +from pathlib import Path + +from ..outputs import Results +from ..project import Project +from ..wrappers import MatlabWrapper + +RUNNER = """function executeRAT() +project = jsonToProject('{project}'); +controls = jsonToControls('{control}'); +[project, results] = RAT(project, controls); + +projectToJson(project, '{project}'); +resultsToJson(results, '{result}'); +end +""" + + +def run_matlab_directly(project, controls, matlab_rat_path): + """Run User provided MATLAB RAT for the given project and controls inputs. + + Parameters + ---------- + project : RAT.Project + The project model, which defines the physical system under study. + controls : RAT.Controls + The controls model, which defines algorithmic properties. + matlab_rat_path : str + The path to MATLAB RAT folder. + """ + if MatlabWrapper.loader is None: + raise ImportError(MatlabWrapper.loader_error_message) from None + + engine = MatlabWrapper.loader.result() + cur_dir = os.getcwd() + + with tempfile.TemporaryDirectory() as tmp: + project_file = Path(tmp, "project.json") + control_file = Path(tmp, "controls.json") + result_file = Path(tmp, "results.json") + runner_file = Path(tmp, "executeRAT.m") + + with open(runner_file, "w") as f: + f.write(RUNNER.format(project=project_file, control=control_file, result=result_file)) + + with warnings.catch_warnings(): # Avoid warning about relative paths + warnings.simplefilter("ignore") + project.save(project_file) + controls.save(control_file) + + engine.cd(matlab_rat_path, nargout=0) + engine.eval("addPaths", nargout=0) + engine.cd(cur_dir, nargout=0) + + engine.addpath(tmp, nargout=0) + for file in project.custom_files: + engine.addpath(str(file.path), nargout=0) + + engine.executeRAT(nargout=0) + + project = Project.load(project_file) + results = Results.load(result_file) + return project, results From de0123b21d598b81a836dac0127642fb29551906 Mon Sep 17 00:00:00 2001 From: Stephen Nneji Date: Fri, 15 May 2026 09:26:04 +0100 Subject: [PATCH 2/6] refactor matlab.py --- ratapi/utils/matlab.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/ratapi/utils/matlab.py b/ratapi/utils/matlab.py index dad8921b..14c340cf 100644 --- a/ratapi/utils/matlab.py +++ b/ratapi/utils/matlab.py @@ -1,6 +1,5 @@ """Runs RAT from the MATLAB API.""" -import os import tempfile import warnings from pathlib import Path @@ -10,8 +9,17 @@ from ..wrappers import MatlabWrapper RUNNER = """function executeRAT() + +cur_dir = pwd; +cd('{rat_path}'); +addPaths; +cd(cur_dir); + project = jsonToProject('{project}'); controls = jsonToControls('{control}'); +for i=1:project.customFile.rowCount + addpath(project.customFile.varTable{{i, 5}}); +end [project, results] = RAT(project, controls); projectToJson(project, '{project}'); @@ -36,7 +44,6 @@ def run_matlab_directly(project, controls, matlab_rat_path): raise ImportError(MatlabWrapper.loader_error_message) from None engine = MatlabWrapper.loader.result() - cur_dir = os.getcwd() with tempfile.TemporaryDirectory() as tmp: project_file = Path(tmp, "project.json") @@ -45,22 +52,18 @@ def run_matlab_directly(project, controls, matlab_rat_path): runner_file = Path(tmp, "executeRAT.m") with open(runner_file, "w") as f: - f.write(RUNNER.format(project=project_file, control=control_file, result=result_file)) + f.write( + RUNNER.format(project=project_file, control=control_file, result=result_file, rat_path=matlab_rat_path) + ) with warnings.catch_warnings(): # Avoid warning about relative paths warnings.simplefilter("ignore") project.save(project_file) controls.save(control_file) - engine.cd(matlab_rat_path, nargout=0) - engine.eval("addPaths", nargout=0) - engine.cd(cur_dir, nargout=0) - engine.addpath(tmp, nargout=0) - for file in project.custom_files: - engine.addpath(str(file.path), nargout=0) - engine.executeRAT(nargout=0) + engine.rmpath(tmp, nargout=0) project = Project.load(project_file) results = Results.load(result_file) From 928a034f5ec6fb82bd09a5069988080cac7436f1 Mon Sep 17 00:00:00 2001 From: Stephen Nneji Date: Wed, 20 May 2026 09:59:45 +0100 Subject: [PATCH 3/6] Add support for live plot --- ratapi/project.py | 29 ++++++++++++++++++++--------- ratapi/utils/matlab.py | 31 ++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/ratapi/project.py b/ratapi/project.py index b8ca49f2..cc269e3b 100644 --- a/ratapi/project.py +++ b/ratapi/project.py @@ -927,16 +927,14 @@ def classlist_script(name, classlist): + "\n)" ) - def save(self, filepath: str | Path = "./project.json"): - """Save a project to a JSON file. + def model_dump(self): + """Generate a dictionary representation of the model. - Parameters - ---------- - filepath : str or Path - The path to where the project file will be written. + Returns + ------- + json_dict : dict + A dict containing the model information. """ - filepath = Path(filepath).with_suffix(".json") - json_dict = {} for field in self.model_fields: attr = getattr(self, field) @@ -960,7 +958,7 @@ def make_custom_file_dict(item): "name": item.name, "filename": item.filename, "language": item.language, - "path": try_relative_to(item.path, filepath.parent), + "path": str(item.path), } if item.name != item.function_name: file_dict["function_name"] = item.function_name @@ -973,7 +971,20 @@ def make_custom_file_dict(item): json_dict[field] = [item.model_dump() for item in attr] else: json_dict[field] = attr + return json_dict + def save(self, filepath: str | Path = "./project.json"): + """Save a project to a JSON file. + + Parameters + ---------- + filepath : str or Path + The path to where the project file will be written. + """ + filepath = Path(filepath).with_suffix(".json") + json_dict = self.model_dump() + for file in json_dict["custom_files"]: + file["path"] = try_relative_to(file["path"], filepath.parent) filepath.write_text(json.dumps(json_dict)) @classmethod diff --git a/ratapi/utils/matlab.py b/ratapi/utils/matlab.py index 14c340cf..04a45507 100644 --- a/ratapi/utils/matlab.py +++ b/ratapi/utils/matlab.py @@ -1,5 +1,6 @@ """Runs RAT from the MATLAB API.""" +import json import tempfile import warnings from pathlib import Path @@ -14,9 +15,13 @@ cd('{rat_path}'); addPaths; cd(cur_dir); - + project = jsonToProject('{project}'); controls = jsonToControls('{control}'); +if any(strcmpi(controls.procedure, {{procedures.DE.value, procedures.Simplex.value}})) + disp("hello") + useLivePlot(1); +end for i=1:project.customFile.rowCount addpath(project.customFile.varTable{{i, 5}}); end @@ -24,21 +29,26 @@ projectToJson(project, '{project}'); resultsToJson(results, '{result}'); +close all end """ -def run_matlab_directly(project, controls, matlab_rat_path): +def run_matlab_directly(project, controls, matlab_rat_path, stdout=None, stderr=None): """Run User provided MATLAB RAT for the given project and controls inputs. Parameters ---------- - project : RAT.Project - The project model, which defines the physical system under study. - controls : RAT.Controls - The controls model, which defines algorithmic properties. + project : RAT.Project or dict + The project model (or equivalent json dict), which defines the physical system under study. + controls : RAT.Controls or dict + The controls model (or equivalent json dict), which defines algorithmic properties. matlab_rat_path : str The path to MATLAB RAT folder. + stdout : io.TextIOBase, optional + Text stream for MATLAB console output + stderr : io.TextIOBase, optional + Text stream for MATLAB console error output """ if MatlabWrapper.loader is None: raise ImportError(MatlabWrapper.loader_error_message) from None @@ -56,13 +66,16 @@ def run_matlab_directly(project, controls, matlab_rat_path): RUNNER.format(project=project_file, control=control_file, result=result_file, rat_path=matlab_rat_path) ) + controls.save(control_file) if not isinstance(controls, dict) else control_file.write_text(json.dumps(controls)) + with warnings.catch_warnings(): # Avoid warning about relative paths warnings.simplefilter("ignore") - project.save(project_file) - controls.save(control_file) + project.save(project_file) if not isinstance(project, dict) else project_file.write_text( + json.dumps(project) + ) engine.addpath(tmp, nargout=0) - engine.executeRAT(nargout=0) + engine.executeRAT(nargout=0, stdout=stdout, stderr=stderr) engine.rmpath(tmp, nargout=0) project = Project.load(project_file) From d8aa86200e883d21f2c0267116e62b7294839b8d Mon Sep 17 00:00:00 2001 From: Stephen Nneji Date: Wed, 20 May 2026 12:06:50 +0100 Subject: [PATCH 4/6] Use the IPC file generated in python in MATLAB --- ratapi/project.py | 14 +++++++------- ratapi/utils/matlab.py | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/ratapi/project.py b/ratapi/project.py index cc269e3b..2371e24b 100644 --- a/ratapi/project.py +++ b/ratapi/project.py @@ -932,10 +932,10 @@ def model_dump(self): Returns ------- - json_dict : dict + model_dict : dict A dict containing the model information. """ - json_dict = {} + model_dict = {} for field in self.model_fields: attr = getattr(self, field) @@ -949,7 +949,7 @@ def make_data_dict(item): "simulation_range": item.simulation_range, } - json_dict["data"] = [make_data_dict(data) for data in attr] + model_dict["data"] = [make_data_dict(data) for data in attr] elif field == "custom_files": @@ -965,13 +965,13 @@ def make_custom_file_dict(item): return file_dict - json_dict["custom_files"] = [make_custom_file_dict(file) for file in attr] + model_dict["custom_files"] = [make_custom_file_dict(file) for file in attr] elif isinstance(attr, ClassList): - json_dict[field] = [item.model_dump() for item in attr] + model_dict[field] = [item.model_dump() for item in attr] else: - json_dict[field] = attr - return json_dict + model_dict[field] = attr + return model_dict def save(self, filepath: str | Path = "./project.json"): """Save a project to a JSON file. diff --git a/ratapi/utils/matlab.py b/ratapi/utils/matlab.py index 04a45507..0007c959 100644 --- a/ratapi/utils/matlab.py +++ b/ratapi/utils/matlab.py @@ -18,14 +18,16 @@ project = jsonToProject('{project}'); controls = jsonToControls('{control}'); +customControls = customControl(); +customControls.update(controls); +customControls.filePath = '{ipc_path}'; if any(strcmpi(controls.procedure, {{procedures.DE.value, procedures.Simplex.value}})) - disp("hello") useLivePlot(1); end for i=1:project.customFile.rowCount addpath(project.customFile.varTable{{i, 5}}); end -[project, results] = RAT(project, controls); +[project, results] = RAT(project, customControls); projectToJson(project, '{project}'); resultsToJson(results, '{result}'); @@ -34,7 +36,26 @@ """ -def run_matlab_directly(project, controls, matlab_rat_path, stdout=None, stderr=None): +CONTROL = """classdef customControl < controlsClass + properties(Hidden = true) + filePath = '' + end + methods + function update(obj, controls) + propNames = properties(controls); + for i = 1:length(propNames) + obj.(propNames{i}) = controls.(propNames{i}); + end + end + function path = getIPCFilePath(obj) + path = obj.filePath; + end + end +end +""" + + +def run_matlab_directly(project, controls, matlab_rat_path, ipc_path="", stdout=None, stderr=None): """Run User provided MATLAB RAT for the given project and controls inputs. Parameters @@ -45,6 +66,8 @@ def run_matlab_directly(project, controls, matlab_rat_path, stdout=None, stderr= The controls model (or equivalent json dict), which defines algorithmic properties. matlab_rat_path : str The path to MATLAB RAT folder. + ipc_path : str, optional + IPC path for MATLAB to use stdout : io.TextIOBase, optional Text stream for MATLAB console output stderr : io.TextIOBase, optional @@ -60,10 +83,19 @@ def run_matlab_directly(project, controls, matlab_rat_path, stdout=None, stderr= control_file = Path(tmp, "controls.json") result_file = Path(tmp, "results.json") runner_file = Path(tmp, "executeRAT.m") + custom_controls_file = Path(tmp, "customControl.m") + with open(custom_controls_file, "w") as f: + f.write(CONTROL) with open(runner_file, "w") as f: f.write( - RUNNER.format(project=project_file, control=control_file, result=result_file, rat_path=matlab_rat_path) + RUNNER.format( + project=project_file, + control=control_file, + result=result_file, + rat_path=matlab_rat_path, + ipc_path=ipc_path, + ) ) controls.save(control_file) if not isinstance(controls, dict) else control_file.write_text(json.dumps(controls)) From 0942c69ed9d3986d8c5cf6945e661465fe8181b8 Mon Sep 17 00:00:00 2001 From: Stephen Nneji Date: Fri, 5 Jun 2026 15:32:56 +0100 Subject: [PATCH 5/6] Revert model_dump override --- ratapi/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ratapi/project.py b/ratapi/project.py index 2371e24b..4b997a24 100644 --- a/ratapi/project.py +++ b/ratapi/project.py @@ -927,7 +927,7 @@ def classlist_script(name, classlist): + "\n)" ) - def model_dump(self): + def to_dict(self): """Generate a dictionary representation of the model. Returns @@ -982,7 +982,7 @@ def save(self, filepath: str | Path = "./project.json"): The path to where the project file will be written. """ filepath = Path(filepath).with_suffix(".json") - json_dict = self.model_dump() + json_dict = self.to_dict() for file in json_dict["custom_files"]: file["path"] = try_relative_to(file["path"], filepath.parent) filepath.write_text(json.dumps(json_dict)) From ca25b98de6fa9c5f18ec90797ae408a68c6cb4a1 Mon Sep 17 00:00:00 2001 From: Stephen Nneji Date: Mon, 20 Jul 2026 10:15:11 +0100 Subject: [PATCH 6/6] Add matlab file events --- ratapi/utils/matlab.py | 132 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 4 deletions(-) diff --git a/ratapi/utils/matlab.py b/ratapi/utils/matlab.py index 0007c959..e312d59e 100644 --- a/ratapi/utils/matlab.py +++ b/ratapi/utils/matlab.py @@ -5,6 +5,9 @@ import warnings from pathlib import Path +import numpy as np + +from ..events import EventTypes, PlotEventData, ProgressEventData, notify from ..outputs import Results from ..project import Project from ..wrappers import MatlabWrapper @@ -21,16 +24,18 @@ customControls = customControl(); customControls.update(controls); customControls.filePath = '{ipc_path}'; -if any(strcmpi(controls.procedure, {{procedures.DE.value, procedures.Simplex.value}})) - useLivePlot(1); -end + for i=1:project.customFile.rowCount addpath(project.customFile.varTable{{i, 5}}); end +eventManager.register(eventTypes.Message, @(x) logger('{msg_log_path}', x)); +eventManager.register(eventTypes.Progress, @(x) logger('{progress_log_path}', x)); +eventManager.register(eventTypes.Plot, @(x) logger('{plot_log_path}', x)); [project, results] = RAT(project, customControls); projectToJson(project, '{project}'); resultsToJson(results, '{result}'); +eventManager.clear(); close all end """ @@ -54,6 +59,72 @@ end """ +LOGGER = """function logger(logPath, data) + fid = fopen(logPath, "a"); + cleanup = onCleanup(@() fclose(fid)); + if isstruct(data) + entry = plotDataToJson(data); + elseif iscell(data) + entry = [data{1}, ',', num2str(data{2})]; + else + entry = strip(data, 'right'); + end + fprintf(fid, "%s\\n", entry); +end + +function encoded = plotDataToJson(data) + % Encodes the results into a json file... + tmpResults = struct(); + for fn = fieldnames(data)' + tmpResults.(fn{1}) = data.(fn{1}); + end + + tmpResults.reflectivity = correctCellArray(tmpResults.reflectivity); + tmpResults.shiftedData = correctCellArray(tmpResults.shiftedData); + tmpResults.sldProfiles = makeCellJson(tmpResults.sldProfiles); + tmpResults.resampledLayers = makeCellJson(tmpResults.resampledLayers); + + encoded = jsonencode(tmpResults,ConvertInfAndNaN=false); + encoded = strrep(encoded, ']"', ']'); + encoded = strrep(encoded, '"[', '['); +end + +function outputArray = makeCellJson(cellArray) + % The jsonencode function flattens 2d cell arrays this is a workaround to + % avoid flattening by converting to a string array with is not flattened. + [row, col] = size(cellArray, [1, 2]); + outputArray = strings([row, col]); + for i=1:row + for j=1:col + entry = cellArray{i, j}; + if size(entry, 1) == 1 + entry = {entry}; + end + if col == 1 + entry = {entry}; + end + outputArray(i, j) = jsonencode(entry); + end + end + if row == 1 + outputArray = {outputArray}; + end + +end + +function cellArray = correctCellArray(cellArray) + % Corrects array with single row so its written as 2D array in json + [row, col] = size(cellArray, [1, 2]); + for i=1:row + for j=1:col + if size(cellArray{i, j}, 1) == 1 + cellArray{i, j} = {cellArray{i, j}}; + end + end + end +end +""" + def run_matlab_directly(project, controls, matlab_rat_path, ipc_path="", stdout=None, stderr=None): """Run User provided MATLAB RAT for the given project and controls inputs. @@ -84,6 +155,10 @@ def run_matlab_directly(project, controls, matlab_rat_path, ipc_path="", stdout= result_file = Path(tmp, "results.json") runner_file = Path(tmp, "executeRAT.m") custom_controls_file = Path(tmp, "customControl.m") + msg_log_file = Path(tmp, "runner_msg_log.txt") + progress_log_file = Path(tmp, "runner_progress_log.txt") + plot_log_file = Path(tmp, "runner_plot_log.txt") + Path(tmp, "logger.m").write_text(LOGGER) with open(custom_controls_file, "w") as f: f.write(CONTROL) @@ -95,6 +170,9 @@ def run_matlab_directly(project, controls, matlab_rat_path, ipc_path="", stdout= result=result_file, rat_path=matlab_rat_path, ipc_path=ipc_path, + msg_log_path=msg_log_file, + progress_log_path=progress_log_file, + plot_log_path=plot_log_file, ) ) @@ -107,8 +185,54 @@ def run_matlab_directly(project, controls, matlab_rat_path, ipc_path="", stdout= ) engine.addpath(tmp, nargout=0) - engine.executeRAT(nargout=0, stdout=stdout, stderr=stderr) + future = engine.executeRAT(nargout=0, stdout=stdout, stderr=stdout, background=True) + msg_cur_line = 0 + plot_cur_line = 0 + progress_cur_line = 0 + while not future.done(): + if msg_log_file.exists(): + with open(msg_log_file, encoding="utf-8") as handle: + handle.seek(msg_cur_line) + text = handle.read() + msg_cur_line = handle.tell() + if text: + notify(EventTypes.Message, text) + if progress_log_file.exists(): + with open(progress_log_file, encoding="utf-8") as handle: + handle.seek(progress_cur_line) + lines = handle.readlines() + progress_cur_line = handle.tell() + if lines: + msg, percent = lines[-1].strip().rsplit(",", 1) + progress_data = ProgressEventData() + progress_data.message = msg + progress_data.percent = float(percent) + notify(EventTypes.Progress, progress_data) + if plot_log_file.exists(): + with open(plot_log_file, encoding="utf-8") as handle: + handle.seek(plot_cur_line) + lines = handle.readlines() + plot_cur_line = handle.tell() + if lines: + plot_data = PlotEventData() + plot_json = json.loads(lines[-1]) + plot_data.modelType = plot_json["modelType"] + plot_data.reflectivity = [np.array(ref) for ref in plot_json["reflectivity"]] + plot_data.shiftedData = [np.array(sd) for sd in plot_json["shiftedData"]] + plot_data.sldProfiles = [ + [np.array(prof) for prof in profiles] for profiles in plot_json["sldProfiles"] + ] + plot_data.resampledLayers = [ + [np.array(lay) for lay in layers] for layers in plot_json["resampledLayers"] + ] + plot_data.dataPresent = plot_json["dataPresent"] + plot_data.subRoughs = plot_json["subRoughs"] + plot_data.resample = plot_json["resample"] + plot_data.contrastNames = plot_json["contrastNames"] + notify(EventTypes.Plot, plot_data) engine.rmpath(tmp, nargout=0) + if future.result() is not None: + raise RuntimeError(future.result()) project = Project.load(project_file) results = Results.load(result_file)