From 2d5439964a09e5ae3742371107a95e7111065126 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 23 Oct 2025 11:27:37 +0100 Subject: [PATCH 01/18] #32: Basic prototype callback addition --- lib/ants/fileformats/ancil/__init__.py | 36 +++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index fb1546d..76c7fc7 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -21,6 +21,7 @@ cube.attributes['grid_staggering']. """ +import glob import ants import iris import mule @@ -50,12 +51,16 @@ def __init__(self, grid_staggering): def __call__(self, cube, field, filename): """ - ANTS callback to add grid staggering and maintain pseudo level order. + ANTS callback to add grid staggering, maintain pseudo level order. Used as a callback when loading fields files for all ants.io.load operations (e.g. :func:`~ants.io.load.load`, :func:`~ants.io.load.load_cube` etc). + This callback will load additional metadata files with the naming convention: + filename. and append the contents of those files to the cube + attributes. + Parameters ---------- cube : :class:`iris.cube.Cube` @@ -68,8 +73,32 @@ def __call__(self, cube, field, filename): """ cube.attributes["grid_staggering"] = self.grid_staggering[filename] + metadata_files = glob.glob(filename+".*") + if metadata_files: + self._retrieve_metadata(metadata_files, cube) super(_CallbackUM, self).__call__(cube, field, filename) + def _retrieve_metadata(self, metadata_files, cube): + """ + Reads in the contents of each file and adds an attribute to the cube with the + name of the file. + + Parameters + ---------- + metadata_files: list + The list of filenames that contain metadata for the cube. + cube : :class:`iris.cube.Cube` + The cube being loaded. + + """ + for metadata_file in metadata_files: + open_file = open(metadata_file, "r") + metadata = open_file.readlines() + open_file.close() + file_name_splits = str(metadata_file).split(".") + attribute_name = file_name_splits[-1] + cube.attributes[attribute_name] = metadata + class _IrisPPFieldDataProvider(object): def __init__(self, ppfield): @@ -345,8 +374,13 @@ def load_cubes(*args, **kwargs): :func:`iris.fileformats.um.load_cubes` """ + #change this function grid_staggering = _fetch_grid_staggering_from_file(args[0]) args, kwargs = pp._add_callback(_CallbackUM(grid_staggering), *args, **kwargs) + # here + #get the filepath - args[0]? search for files of the same name +. + #load in contents as attribute dictionary - in callback? + # add to cube return iris.fileformats.um.load_cubes(*args, **kwargs) From 116ddb20866c4b730740c5065ee271f983080243 Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Wed, 12 Nov 2025 14:23:25 +0000 Subject: [PATCH 02/18] #32: update pytest tempdir config --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ab7ed41..e5cbaf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ skip = ".git" # Configure pytest. [tool.pytest.ini_options] addopts = "--durations=5" +tmp_path_retention_policy = "none" filterwarnings = [ # During unittesting only, treat all warnings as errors by default: 'error', From ab3f3cca5d127d604f239bfa1f701790f11faf52 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:12:41 +0000 Subject: [PATCH 03/18] #32: Working prototype --- lib/ants/io/load.py | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 6aa48d9..46d732b 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -57,6 +57,7 @@ """ import copy import warnings +import glob from contextlib import contextmanager from functools import wraps @@ -354,6 +355,23 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) + user_callback = None + if len(args)>1: + user_callback = args[1] + else: + if 'callback' in kwargs: + user_callback = kwargs.pop('callback') + if 'ignore_metadata_files' in kwargs: + ignore_metadata_files = kwargs.pop('ignore_metadata_files') + #Do the handling for each way a user callback can be passed in through iris + if ignore_metadata_files == False: + args, kwargs = _add_metadata_file_callback(user_callback, args, kwargs) + #_create_callabck_metadata + #do the create callback function and append to callbacks + # (copy pp callbak if user callback exists then use the existing + # pp callback code to append that to the ants callback) + else: + args, kwargs = _add_callback(_CallbackMetadata(user_callback), *args, **kwargs) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): cubes = func(*args, **kwargs) @@ -368,6 +386,75 @@ def load_function(*args, **kwargs): return load_function +def _add_metadata_file_callback(user_callback, *args, **kwargs): + """ + have a seperate function for loading metadata + if user callback is not none + do an update the same way as in the pp file + copy pp callback to be the same and add user one to run after this + + """ + args, kwargs = _add_callback(_CallbackMetadata(), *args, **kwargs) + return args, kwargs + +def _add_callback(callback, *args, **kwargs): + """ + Adds both the ants callback and the user provided callback (if any) to the + load. + + """ + args = list(args) + if len(args) == 1: + kwargs['callback'] = callback + else: + args[2] = callback + args = tuple(args) + return args, kwargs + +class _CallbackMetadata(object): + """Callback for collecting metadata from sidecar files. + + This callback will load additional metadata files with the naming convention: + filename. and append the contents of those files to the cube + attributes. + """ + def __init__(self, user_callback): + self._user_callback = user_callback + + def __call__(self, cube, field, filename): + """ + + """ + metadata_filenames = ''.join([filename, ".*"]) + metadata_files = glob.glob(metadata_filenames) + if metadata_files != []: + self._retrieve_metadata(metadata_files, cube) + if self._user_callback is not None: + self._user_callback(cube, field, filename) + + def _retrieve_metadata(self, metadata_files, cube): + """ + Reads in the contents of each file and adds an attribute to the cube with the + name of the file. + + Parameters + ---------- + metadata_files: list + The list of filenames that contain metadata for the cube. + cube : :class:`iris.cube.Cube` + The cube being loaded. + + """ + for metadata_file in metadata_files: + open_file = open(metadata_file, "r") + metadata = open_file.readlines() + open_file.close() + file_name_splits = str(metadata_file).split(".") + attribute_name = file_name_splits[-1] + cube.attributes[attribute_name] = metadata + with open('written_license.txt', 'a')as file: + file.write(''.join(metadata)) + def load_cube(*args, **kwargs): """ From 9af1590eaa1f9d1537fe8efc56c95ec9088d3fb0 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 13 Nov 2025 17:00:45 +0000 Subject: [PATCH 04/18] #32: Working prototype with improved logic --- lib/ants/io/load.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 46d732b..605cac0 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -355,22 +355,19 @@ def load_function(*args, **kwargs): "iris.FUTURE.datum_support flag.", FutureWarning, ) - user_callback = None - if len(args)>1: - user_callback = args[1] - else: - if 'callback' in kwargs: - user_callback = kwargs.pop('callback') + ignore_metadata_files = False if 'ignore_metadata_files' in kwargs: ignore_metadata_files = kwargs.pop('ignore_metadata_files') - #Do the handling for each way a user callback can be passed in through iris - if ignore_metadata_files == False: - args, kwargs = _add_metadata_file_callback(user_callback, args, kwargs) - #_create_callabck_metadata - #do the create callback function and append to callbacks - # (copy pp callbak if user callback exists then use the existing - # pp callback code to append that to the ants callback) - else: + if ignore_metadata_files == False: + #Do the handling for each way a user callback can be passed in through iris + user_callback = None + if len(args)>1: + user_callback = args[1] + else: + if 'callback' in kwargs: + user_callback = kwargs.pop('callback') + print("first args: ", args) + print("first kwargs: ", kwargs) args, kwargs = _add_callback(_CallbackMetadata(user_callback), *args, **kwargs) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): @@ -386,17 +383,6 @@ def load_function(*args, **kwargs): return load_function -def _add_metadata_file_callback(user_callback, *args, **kwargs): - """ - have a seperate function for loading metadata - if user callback is not none - do an update the same way as in the pp file - copy pp callback to be the same and add user one to run after this - - """ - args, kwargs = _add_callback(_CallbackMetadata(), *args, **kwargs) - return args, kwargs - def _add_callback(callback, *args, **kwargs): """ Adds both the ants callback and the user provided callback (if any) to the @@ -404,11 +390,17 @@ def _add_callback(callback, *args, **kwargs): """ args = list(args) + print(args) + print("len args: ", len(args)) if len(args) == 1: kwargs['callback'] = callback + elif len(args) == 2: + args[1] = callback else: args[2] = callback args = tuple(args) + print("with first callback args: ", args) + print("with first callback kwargs: ", kwargs) return args, kwargs class _CallbackMetadata(object): @@ -425,6 +417,7 @@ def __call__(self, cube, field, filename): """ """ + print("callback has been added") metadata_filenames = ''.join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) if metadata_files != []: From dff558e3ad81a0058e82963cd2870158bd84029a Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 09:35:14 +0000 Subject: [PATCH 05/18] #32: Update argument handler to correctly pass in user callback --- lib/ants/io/load.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 605cac0..84c3339 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -361,8 +361,8 @@ def load_function(*args, **kwargs): if ignore_metadata_files == False: #Do the handling for each way a user callback can be passed in through iris user_callback = None - if len(args)>1: - user_callback = args[1] + if len(args)==3: + user_callback = args[2] else: if 'callback' in kwargs: user_callback = kwargs.pop('callback') @@ -394,9 +394,7 @@ def _add_callback(callback, *args, **kwargs): print("len args: ", len(args)) if len(args) == 1: kwargs['callback'] = callback - elif len(args) == 2: - args[1] = callback - else: + elif len(args) == 3: args[2] = callback args = tuple(args) print("with first callback args: ", args) @@ -423,6 +421,7 @@ def __call__(self, cube, field, filename): if metadata_files != []: self._retrieve_metadata(metadata_files, cube) if self._user_callback is not None: + print("should have added user callback") self._user_callback(cube, field, filename) def _retrieve_metadata(self, metadata_files, cube): From de3d650d10c4c3cba312ba7b47eb98962d52c9ba Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 10:02:33 +0000 Subject: [PATCH 06/18] #32: revert changes to the ancil loading --- lib/ants/fileformats/ancil/__init__.py | 34 -------------------------- 1 file changed, 34 deletions(-) diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index 76c7fc7..abc31bc 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -57,10 +57,6 @@ def __call__(self, cube, field, filename): operations (e.g. :func:`~ants.io.load.load`, :func:`~ants.io.load.load_cube` etc). - This callback will load additional metadata files with the naming convention: - filename. and append the contents of those files to the cube - attributes. - Parameters ---------- cube : :class:`iris.cube.Cube` @@ -73,33 +69,8 @@ def __call__(self, cube, field, filename): """ cube.attributes["grid_staggering"] = self.grid_staggering[filename] - metadata_files = glob.glob(filename+".*") - if metadata_files: - self._retrieve_metadata(metadata_files, cube) super(_CallbackUM, self).__call__(cube, field, filename) - def _retrieve_metadata(self, metadata_files, cube): - """ - Reads in the contents of each file and adds an attribute to the cube with the - name of the file. - - Parameters - ---------- - metadata_files: list - The list of filenames that contain metadata for the cube. - cube : :class:`iris.cube.Cube` - The cube being loaded. - - """ - for metadata_file in metadata_files: - open_file = open(metadata_file, "r") - metadata = open_file.readlines() - open_file.close() - file_name_splits = str(metadata_file).split(".") - attribute_name = file_name_splits[-1] - cube.attributes[attribute_name] = metadata - - class _IrisPPFieldDataProvider(object): def __init__(self, ppfield): self.ppfield = ppfield @@ -374,13 +345,8 @@ def load_cubes(*args, **kwargs): :func:`iris.fileformats.um.load_cubes` """ - #change this function grid_staggering = _fetch_grid_staggering_from_file(args[0]) args, kwargs = pp._add_callback(_CallbackUM(grid_staggering), *args, **kwargs) - # here - #get the filepath - args[0]? search for files of the same name +. - #load in contents as attribute dictionary - in callback? - # add to cube return iris.fileformats.um.load_cubes(*args, **kwargs) From 877a88ebcb1c1ead0ef658a691de91fab14068c8 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:52:05 +0000 Subject: [PATCH 07/18] #32: Add etsts for new loading functionality --- lib/ants/fileformats/ancil/__init__.py | 1 + lib/ants/io/load.py | 43 ++++++---- .../tests/fileformats/pp/test_CallbackPP.py | 5 +- .../tests/io/load/test_CallbackMetadata.py | 83 +++++++++++++++++++ 4 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 lib/ants/tests/io/load/test_CallbackMetadata.py diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index abc31bc..056b6cf 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -71,6 +71,7 @@ def __call__(self, cube, field, filename): cube.attributes["grid_staggering"] = self.grid_staggering[filename] super(_CallbackUM, self).__call__(cube, field, filename) + class _IrisPPFieldDataProvider(object): def __init__(self, ppfield): self.ppfield = ppfield diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 84c3339..6d760e2 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -356,19 +356,27 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False - if 'ignore_metadata_files' in kwargs: - ignore_metadata_files = kwargs.pop('ignore_metadata_files') + if "ignore_metadata_files" in kwargs: + ignore_metadata_files = kwargs.pop("ignore_metadata_files") + print( + "ignore metadata files: ", + ignore_metadata_files, + type(ignore_metadata_files), + ) if ignore_metadata_files == False: - #Do the handling for each way a user callback can be passed in through iris + print("doing the thing") + # Do the handling for each way a user callback can be passed in through iris user_callback = None - if len(args)==3: + if len(args) == 3: user_callback = args[2] else: - if 'callback' in kwargs: - user_callback = kwargs.pop('callback') + if "callback" in kwargs: + user_callback = kwargs.pop("callback") print("first args: ", args) print("first kwargs: ", kwargs) - args, kwargs = _add_callback(_CallbackMetadata(user_callback), *args, **kwargs) + args, kwargs = _add_callback( + _CallbackMetadata(user_callback), *args, **kwargs + ) # Use context manager to avoid permanently modifying iris behaviour. with ants_format_agent(): cubes = func(*args, **kwargs) @@ -383,40 +391,39 @@ def load_function(*args, **kwargs): return load_function + def _add_callback(callback, *args, **kwargs): """ Adds both the ants callback and the user provided callback (if any) to the load. - """ args = list(args) - print(args) - print("len args: ", len(args)) if len(args) == 1: - kwargs['callback'] = callback + kwargs["callback"] = callback elif len(args) == 3: args[2] = callback args = tuple(args) - print("with first callback args: ", args) - print("with first callback kwargs: ", kwargs) return args, kwargs + class _CallbackMetadata(object): """Callback for collecting metadata from sidecar files. - + This callback will load additional metadata files with the naming convention: filename. and append the contents of those files to the cube attributes. """ + def __init__(self, user_callback): self._user_callback = user_callback def __call__(self, cube, field, filename): """ - + The method that runs when iris runs the callback. Collects the filenames and + will run the user callback """ print("callback has been added") - metadata_filenames = ''.join([filename, ".*"]) + metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) if metadata_files != []: self._retrieve_metadata(metadata_files, cube) @@ -444,8 +451,8 @@ def _retrieve_metadata(self, metadata_files, cube): file_name_splits = str(metadata_file).split(".") attribute_name = file_name_splits[-1] cube.attributes[attribute_name] = metadata - with open('written_license.txt', 'a')as file: - file.write(''.join(metadata)) + with open("written_license.txt", "a") as file: + file.write("".join(metadata)) def load_cube(*args, **kwargs): diff --git a/lib/ants/tests/fileformats/pp/test_CallbackPP.py b/lib/ants/tests/fileformats/pp/test_CallbackPP.py index 628299d..a93d964 100644 --- a/lib/ants/tests/fileformats/pp/test_CallbackPP.py +++ b/lib/ants/tests/fileformats/pp/test_CallbackPP.py @@ -4,6 +4,9 @@ # See LICENSE.txt in the root of the repository for full licensing details. import unittest.mock as mock +import tempfile +import os +import pytest import ants.tests import iris @@ -18,7 +21,7 @@ def setUp(self): def call(self): callback = CallbackPP() callback.append_user_callback(self.user_callback) - callback(self.cube, mock.sentinel.field, mock.sentinel.filename) + callback(self.cube, mock.sentinel.field, "filename") def test__freeze_pseudo_level_is_called_if_pseudo_level_present(self): pseudo_level = iris.coords.DimCoord([0], long_name="pseudo_level") diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py new file mode 100644 index 0000000..50a3c5b --- /dev/null +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -0,0 +1,83 @@ +""" +Includes tests for end-to-end functionality of CallbackMetadata as well as calling the +class directly. +""" + +import ants.io.load +import iris +import pytest +import unittest.mock as mock + + +def test_metadata_files_added_to_attributes(tmp_path): + """Tests that metadata files are found and added to the cube's attributes.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + # How the text should look while stored in an array + loaded_license = [ + "This is the license of the cube.\n", + "\n", + " It should be preserved and added to the cube when loaded.\n", + " ", + ] + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_test_cube.attributes["license"] == loaded_license + + +def test_no_metadata_loaded(tmp_path): + """Tests that metadata files are not loaded when the option is turned off.""" + # The text that would be in a license file + license_text = """This is the license of the cube. + + It should be preserved and added to the cube when loaded. + """ + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.pp.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + loaded_test_cube = ants.io.load.load_cube( + temporary_cube_path, ignore_metadata_files=True + ) + with pytest.raises(KeyError): + loaded_test_cube.attributes["license"] + + +def test_user_callback_added(): + """Test that on ititialisation, the user's function will be set.""" + + def user_callback(cube, field, filename): + print("a user's callback, passed in") + + class_instance = ants.io.load._CallbackMetadata(user_callback) + assert class_instance._user_callback == user_callback + + +def test_args_parsed_correctly_with_kwargs(tmp_path): + """Tests that when passed a callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, callback=mock_callback) + mock_callback.assert_called() + + +def test_args_parsed_correctly_with_positional_args(tmp_path): + """Tests that when passed a callback using a keyword argument, + the callback is parsed correctly.""" + mock_callback = mock.Mock() + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.pp" + iris.save(test_cube, str(temporary_cube_path)) + ants.io.load.load_cube(temporary_cube_path, None, mock_callback) + mock_callback.assert_called() From fa39ac105e20ddc8bab7e7405a6921541703dda2 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:53:50 +0000 Subject: [PATCH 08/18] #32: Revert changes to ancil/__init__.py --- lib/ants/fileformats/ancil/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/ants/fileformats/ancil/__init__.py b/lib/ants/fileformats/ancil/__init__.py index 056b6cf..fb1546d 100644 --- a/lib/ants/fileformats/ancil/__init__.py +++ b/lib/ants/fileformats/ancil/__init__.py @@ -21,7 +21,6 @@ cube.attributes['grid_staggering']. """ -import glob import ants import iris import mule @@ -51,7 +50,7 @@ def __init__(self, grid_staggering): def __call__(self, cube, field, filename): """ - ANTS callback to add grid staggering, maintain pseudo level order. + ANTS callback to add grid staggering and maintain pseudo level order. Used as a callback when loading fields files for all ants.io.load operations (e.g. :func:`~ants.io.load.load`, :func:`~ants.io.load.load_cube` From 4c354b716c13d8b84d8781da724eb73b8dcde5d9 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:57:47 +0000 Subject: [PATCH 09/18] #32: Revert pp tests --- lib/ants/tests/fileformats/pp/test_CallbackPP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ants/tests/fileformats/pp/test_CallbackPP.py b/lib/ants/tests/fileformats/pp/test_CallbackPP.py index a93d964..4fd5b03 100644 --- a/lib/ants/tests/fileformats/pp/test_CallbackPP.py +++ b/lib/ants/tests/fileformats/pp/test_CallbackPP.py @@ -21,7 +21,7 @@ def setUp(self): def call(self): callback = CallbackPP() callback.append_user_callback(self.user_callback) - callback(self.cube, mock.sentinel.field, "filename") + callback(self.cube, mock.sentinel.field, mock.sentinel.filename) def test__freeze_pseudo_level_is_called_if_pseudo_level_present(self): pseudo_level = iris.coords.DimCoord([0], long_name="pseudo_level") From 3dc860b794ead77f0365772e8be49e9586ef7d06 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:58:11 +0000 Subject: [PATCH 10/18] #32: Remove unused imports --- lib/ants/tests/fileformats/pp/test_CallbackPP.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/ants/tests/fileformats/pp/test_CallbackPP.py b/lib/ants/tests/fileformats/pp/test_CallbackPP.py index 4fd5b03..628299d 100644 --- a/lib/ants/tests/fileformats/pp/test_CallbackPP.py +++ b/lib/ants/tests/fileformats/pp/test_CallbackPP.py @@ -4,9 +4,6 @@ # See LICENSE.txt in the root of the repository for full licensing details. import unittest.mock as mock -import tempfile -import os -import pytest import ants.tests import iris From da8e951718cffbfc4cc03f8576758e443651ddc3 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Wed, 10 Dec 2025 10:19:40 +0000 Subject: [PATCH 11/18] #32: Commit to save work on adding new load functionality to command line applications --- lib/ants/cli/ancil_2anc.py | 9 +++++---- lib/ants/cli/ancil_fill_n_merge.py | 8 ++++++-- lib/ants/cli/ancil_general_regrid.py | 9 ++++++++- lib/ants/command_parse.py | 12 ++++++++++++ lib/ants/io/load.py | 2 ++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 60deaa1..84e4f54 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -37,16 +37,16 @@ import iris -def load_data(source): +def load_data(source, ignore_metadata_files): with warnings.catch_warnings(): warnings.filterwarnings( "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation ) - cubes = ants.io.load.load(source) + cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) return cubes -def main(source_path, output_path, grid_staggering, netcdf_only): +def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata_files): """ Convert specified source file to an ancillary. @@ -66,7 +66,7 @@ def main(source_path, output_path, grid_staggering, netcdf_only): written to an ancillary. """ - source_cubes = load_data(source_path) + source_cubes = load_data(source_path, ignore_metadata_files) if grid_staggering is not None: for source_cube in source_cubes: source_cube.attributes["grid_staggering"] = grid_staggering @@ -98,6 +98,7 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, + args.ignore_metadata_files ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 9bb7cea..41333ab 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -19,6 +19,7 @@ def load_data( primary_source, + ignore_metadata_files, alternate_source=None, validity_polygon_filepath=None, target_mask_filepath=None, @@ -56,12 +57,12 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load(primary_source) + primary_cubes = ants.io.load.load(primary_source, ignore_metadata_files=ignore_metadata_files) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load(alternate_source) + alternate_cubes = ants.io.load.load(alternate_source,ignore_metadata_files=ignore_metadata_files) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -96,6 +97,7 @@ def main( end, netcdf_only, search_method, + ignore_metadata_files ): """ Perform merge and fill operation on the provided sources. @@ -159,6 +161,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files ) result = primary_cubes @@ -251,6 +254,7 @@ def cli_interface(): end=args.end, netcdf_only=args.netcdf_only, search_method=args.search_method, + ignore_metadata_files=args.ignore_metadata_files ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 7f6bd86..01fe332 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -37,13 +37,14 @@ def load_data( source, + ignore_metadata_files, target_grid=None, target_landseamask=None, land_fraction_threshold=None, begin=None, end=None, ): - source_cubes = ants.io.load.load(source) + source_cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: @@ -76,6 +77,7 @@ def main( save_ukca, netcdf_only, search_method, + ignore_metadata_files, ): """ General regrid application top level call function. @@ -118,6 +120,9 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. + invert_mask : :obj:`bool`, optional + When set to True, files containing metadata will not be loaded alongside data + and added as attributes to the cube. Returns ------- @@ -132,6 +137,7 @@ def main( land_fraction_threshold, begin, end, + ignore_metadata_files, ) if ants.utils.cube._is_ugrid(target_cube): raise ValueError( @@ -210,6 +216,7 @@ def cli_interface(): args.save_ukca, args.netcdf_only, args.search_method, + args.ignore_metadata_files, ) diff --git a/lib/ants/command_parse.py b/lib/ants/command_parse.py index 2a3a01c..3a58ed3 100644 --- a/lib/ants/command_parse.py +++ b/lib/ants/command_parse.py @@ -33,6 +33,11 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): sources ... Source data path(s). --output , -o Output filepath --ants-config Configuration path. + --ignore-metadata-files Stops the application from + searching for metadata files + with the same name as the + source and loading them + alongside the source. Additionally, there are standardised optional arguments and these are activated by passing the relevant keyword argument. See 'Parameters' @@ -171,6 +176,13 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False): help="Only write out a netCDF file.", required=False, ) + self.add_argument( + "--ignore-metadata-files", + action="store_true", + help="Turn off the automatic loading of metadata files alongside source " + "files", + required=False, + ) if time_constraints: self.add_argument( "--begin", diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index ee54bc4..155c2f7 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -357,6 +357,8 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False + print("args: ", args) + print("kwargs: ", kwargs) if "ignore_metadata_files" in kwargs: ignore_metadata_files = kwargs.pop("ignore_metadata_files") print( From 79793eed4dd947f9a8ed7095c56a23a8a26bb9cf Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Mon, 5 Jan 2026 12:42:58 +0000 Subject: [PATCH 12/18] #32: Working rose stem with new load functionality --- lib/ants/cli/ancil_general_regrid.py | 7 ++++--- lib/ants/io/load.py | 12 ++++++++---- rose-stem/app/ancil_2anc/rose-app.conf | 2 +- rose-stem/app/ancil_create_shapefile/rose-app.conf | 2 +- .../ancil_fill_n_merge/opt/rose-app-invert_mask.conf | 2 +- .../ancil_fill_n_merge/opt/rose-app-land_cover.conf | 2 +- .../opt/rose-app-grid_to_grid.conf | 2 +- .../opt/rose-app-invert_mask.conf | 2 +- rose-stem/app/ancil_general_regrid/rose-app.conf | 2 +- .../rose-app.conf | 2 +- 10 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 01fe332..94ec6e1 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -37,18 +37,19 @@ def load_data( source, - ignore_metadata_files, target_grid=None, target_landseamask=None, land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None, ): source_cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid(target_grid) + target_cube = ants.io.load.load_grid(target_grid, + ignore_metadata_files=ignore_metadata_files) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold @@ -120,7 +121,7 @@ def main( provided source(s) consistent with the provided land sea mask. This should only be provided if a target land sea mask is also provided via target_lsm_path. - invert_mask : :obj:`bool`, optional + ignore_metadata_files : :obj:`bool`, optional When set to True, files containing metadata will not be loaded alongside data and added as attributes to the cube. diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 155c2f7..5dca7d9 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -232,18 +232,18 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask") + lbm = ants.io.load.load_cube(filename, "land_binary_mask", ignore_metadata_files=True) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction") + land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction",ignore_metadata_files=True) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: # It looks like we are wanting to extract a landsea mask from some # other field. - cube = ants.io.load.load(filename)[0] + cube = ants.io.load.load(filename, ignore_metadata_files=True)[0] y = cube.coord(axis="y") x = cube.coord(axis="x") cube = cube.slices((y, x)).next() @@ -357,9 +357,13 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False + print("the func: ", func) print("args: ", args) print("kwargs: ", kwargs) - if "ignore_metadata_files" in kwargs: + print("boo") + print('ignore_metadata_files' in kwargs) + if 'ignore_metadata_files' in kwargs: + print("its here!!!") ignore_metadata_files = kwargs.pop("ignore_metadata_files") print( "ignore metadata files: ", diff --git a/rose-stem/app/ancil_2anc/rose-app.conf b/rose-stem/app/ancil_2anc/rose-app.conf index 7ab4ed0..60f9ee4 100644 --- a/rose-stem/app/ancil_2anc/rose-app.conf +++ b/rose-stem/app/ancil_2anc/rose-app.conf @@ -11,7 +11,7 @@ history=${CYLC_WORKFLOW_ID}@${ROSE_SUITE_REVISION}:${ROSE_TASK_NAME} [command] default=ants-launch ancil_2anc.py \ =${source} --grid-staggering ${grid_staggering} -o ${output} \ - =--ants-config ${ANTS_CONFIG} + =--ants-config ${ANTS_CONFIG} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_create_shapefile/rose-app.conf b/rose-stem/app/ancil_create_shapefile/rose-app.conf index 4f37bbf..ac0bbe0 100644 --- a/rose-stem/app/ancil_create_shapefile/rose-app.conf +++ b/rose-stem/app/ancil_create_shapefile/rose-app.conf @@ -1,5 +1,5 @@ [command] -default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} +default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} --ignore-metadata-files [env] OUTPUT=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf index 5d8e792..88b6cc2 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-invert_mask.conf @@ -2,7 +2,7 @@ default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} -o ${output} \ =--ants-config ${ANTS_CONFIG} --invert-mask \ - =--search-method ${search_method} + =--search-method ${search_method} --ignore-metadata-files [env] search_method=${CYLC_TASK_PARAM_fill} diff --git a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf index 4db75a3..bd71de6 100644 --- a/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf +++ b/rose-stem/app/ancil_fill_n_merge/opt/rose-app-land_cover.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_fill_n_merge.py \ =${source} --target-lsm ${target} --polygon ${polygon} -o ${output} \ - =--ants-config ${ANTS_CONFIG} --search-method ${search_method} + =--ants-config ${ANTS_CONFIG} --search-method ${search_method} --ignore-metadata-files [env] polygon=${ROSE_DATA}/ite.shp diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf index 23bc985..78e6087 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-grid_to_grid.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} --search-method ${search_method} + =${begin} ${end} --search-method ${search_method} --ignore-metadata-files [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf index 9a4c4ba..5a7fd0e 100644 --- a/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf +++ b/rose-stem/app/ancil_general_regrid/opt/rose-app-invert_mask.conf @@ -1,7 +1,7 @@ [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--invert-mask ${begin} ${end} --search-method ${search_method} + =--invert-mask ${begin} ${end} --search-method ${search_method} --ignore-metadata-files [env] output=${ROSE_DATA}/${ROSE_TASK_NAME} diff --git a/rose-stem/app/ancil_general_regrid/rose-app.conf b/rose-stem/app/ancil_general_regrid/rose-app.conf index fb1a949..778e84f 100644 --- a/rose-stem/app/ancil_general_regrid/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =${begin} ${end} + =${begin} ${end} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf diff --git a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf index 6f166bd..ad44222 100644 --- a/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf +++ b/rose-stem/app/ancil_general_regrid_with_time_constraint/rose-app.conf @@ -14,7 +14,7 @@ scheme=TwoStage [command] default=ants-launch ancil_general_regrid.py \ =${source} -o ${output} --ants-config ${ANTS_CONFIG} --target-${target_type} ${target} \ - =--begin ${begin} --end ${end} --search-method ${search_method} + =--begin ${begin} --end ${end} --search-method ${search_method} --ignore-metadata-files [env] ANTS_CONFIG=rose-app-run.conf From 119dfcf8154e12e4857b6f30592513ff71a80f69 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:16:50 +0000 Subject: [PATCH 13/18] #32: Run black --- lib/ants/cli/ancil_2anc.py | 2 +- lib/ants/cli/ancil_fill_n_merge.py | 14 +++++++++----- lib/ants/cli/ancil_general_regrid.py | 9 ++++++--- lib/ants/io/load.py | 12 ++++++++---- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/lib/ants/cli/ancil_2anc.py b/lib/ants/cli/ancil_2anc.py index 84e4f54..48a6f05 100755 --- a/lib/ants/cli/ancil_2anc.py +++ b/lib/ants/cli/ancil_2anc.py @@ -98,7 +98,7 @@ def cli_interface(): args.output, args.grid_staggering, args.netcdf_only, - args.ignore_metadata_files + args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 41333ab..4f15bd1 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -57,12 +57,16 @@ def load_data( respectively. """ - primary_cubes = ants.io.load.load(primary_source, ignore_metadata_files=ignore_metadata_files) + primary_cubes = ants.io.load.load( + primary_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end) alternate_cubes = None if alternate_source: - alternate_cubes = ants.io.load.load(alternate_source,ignore_metadata_files=ignore_metadata_files) + alternate_cubes = ants.io.load.load( + alternate_source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end) @@ -97,7 +101,7 @@ def main( end, netcdf_only, search_method, - ignore_metadata_files + ignore_metadata_files, ): """ Perform merge and fill operation on the provided sources. @@ -161,7 +165,7 @@ def main( land_fraction_threshold, begin, end, - ignore_metadata_files + ignore_metadata_files, ) result = primary_cubes @@ -254,7 +258,7 @@ def cli_interface(): end=args.end, netcdf_only=args.netcdf_only, search_method=args.search_method, - ignore_metadata_files=args.ignore_metadata_files + ignore_metadata_files=args.ignore_metadata_files, ) diff --git a/lib/ants/cli/ancil_general_regrid.py b/lib/ants/cli/ancil_general_regrid.py index 94ec6e1..dd84790 100755 --- a/lib/ants/cli/ancil_general_regrid.py +++ b/lib/ants/cli/ancil_general_regrid.py @@ -44,12 +44,15 @@ def load_data( end=None, ignore_metadata_files=None, ): - source_cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files) + source_cubes = ants.io.load.load( + source, ignore_metadata_files=ignore_metadata_files + ) if begin is not None: source_cubes = create_time_constrained_cubes(source_cubes, begin, end) if target_grid: - target_cube = ants.io.load.load_grid(target_grid, - ignore_metadata_files=ignore_metadata_files) + target_cube = ants.io.load.load_grid( + target_grid, ignore_metadata_files=ignore_metadata_files + ) else: target_cube = ants.io.load.load_landsea_mask( target_landseamask, land_fraction_threshold diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 5dca7d9..1a69025 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -232,12 +232,16 @@ def load_landsea_mask(filename, land_threshold=None): """ try: # Is it a landsea mask field? - lbm = ants.io.load.load_cube(filename, "land_binary_mask", ignore_metadata_files=True) + lbm = ants.io.load.load_cube( + filename, "land_binary_mask", ignore_metadata_files=True + ) lbm = lbm.copy(lbm.data.astype("bool", copy=False)) except iris.exceptions.ConstraintMismatchError: try: # Is it a land fraction field? - land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction",ignore_metadata_files=True) + land_fraction = ants.io.load.load_cube( + filename, "vegetation_area_fraction", ignore_metadata_files=True + ) lbm = land_fraction.copy(land_fraction.data > land_threshold) lbm.rename("land_binary_mask") except iris.exceptions.ConstraintMismatchError: @@ -361,8 +365,8 @@ def load_function(*args, **kwargs): print("args: ", args) print("kwargs: ", kwargs) print("boo") - print('ignore_metadata_files' in kwargs) - if 'ignore_metadata_files' in kwargs: + print("ignore_metadata_files" in kwargs) + if "ignore_metadata_files" in kwargs: print("its here!!!") ignore_metadata_files = kwargs.pop("ignore_metadata_files") print( From 04b3f9c2359f3c9d4ce2c4149c87b9533a390d7e Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:17:23 +0000 Subject: [PATCH 14/18] #32: Remove unneeded argument flag --- rose-stem/app/ancil_create_shapefile/rose-app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rose-stem/app/ancil_create_shapefile/rose-app.conf b/rose-stem/app/ancil_create_shapefile/rose-app.conf index ac0bbe0..4f37bbf 100644 --- a/rose-stem/app/ancil_create_shapefile/rose-app.conf +++ b/rose-stem/app/ancil_create_shapefile/rose-app.conf @@ -1,5 +1,5 @@ [command] -default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} --ignore-metadata-files +default=ants-launch ancil_create_shapefile.py ${SHAPEFILE_JSON} ${OUTPUT} [env] OUTPUT=${ROSE_DATA}/ite.shp From abcdeae83cc29145609835d1739c8618fec20f9e Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 6 Jan 2026 09:38:27 +0000 Subject: [PATCH 15/18] #32: Fix to work with unittests --- lib/ants/cli/ancil_fill_n_merge.py | 2 +- lib/ants/io/load.py | 2 ++ .../tests/command_parse/test_integration.py | 34 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 4f15bd1..3fee185 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -19,13 +19,13 @@ def load_data( primary_source, - ignore_metadata_files, alternate_source=None, validity_polygon_filepath=None, target_mask_filepath=None, land_fraction_threshold=None, begin=None, end=None, + ignore_metadata_files=None ): """ Load the necessary data for performing a merge and fill operation. diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 1a69025..0753226 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -434,6 +434,8 @@ def __call__(self, cube, field, filename): will run the user callback """ print("callback has been added") + if type(filename) is list: + filename = filename[0] metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) if metadata_files != []: diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index 31105ab..cd5f50c 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -30,6 +30,7 @@ def setUp(self): self.addCleanup(patch.stop) def test_default_args(self): + """Tests that the default arguments are as expected.""" new = [ "program", "/path/to/source", @@ -53,6 +54,7 @@ def path_check(filename, **kwargs): output="/path/to/output", sources=["/path/to/source"], netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -69,6 +71,7 @@ def test_add_args(self): output="/path/to/output", ants_config=None, netcdf_only=False, + ignore_metadata_files=False, new_arg="new_arg_value", ) self.assertEqual(args, target_args) @@ -150,6 +153,7 @@ def test_invalid_output_same_file(self): self.mock_dirpath_writeable.assert_called_once() def test_time_constraint_args(self): + """Tests that using time constraint arguments will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -176,11 +180,13 @@ def test_time_constraint_args(self): begin=2016, end=2021, netcdf_only=False, + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) def test_time_constraint_flags(self): + """Tests that using time constraint flags will be correctly parsed.""" new = [ "program", "/path/to/source", @@ -207,6 +213,7 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, + ignore_metadata_files=False ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -265,3 +272,30 @@ def test_both_time_constraints_exist(self): parser = AntsArgParser(target_lsm=True, time_constraints=True) with self.assertRaises(exceptions.TimeConstraintMissingException): parser.parse_args() + + def test_set_ignore_metadata_files_flag(self): + "Tests that ignore_metadata_files will be set to True when passed in." + new = [ + "program", + "/path/to/source", + "--target-lsm", + "/path/to/lsm", + "-o", + "/path/to/output", + "--ignore-metadata-files" + ] + with mock.patch("sys.argv", new=new): + parser = AntsArgParser(target_lsm=True) + args = parser.parse_args() + + target_args = argparse.Namespace( + ants_config=None, + land_threshold=None, + target_lsm="/path/to/lsm", + output="/path/to/output", + sources=["/path/to/source"], + netcdf_only=False, + ignore_metadata_files=True, + ) + self.assertFalse(self.mock_config.called) + self.assertEqual(args, target_args) From 212dd0bb47e3f685fe666f735ee81a332432b037 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Thu, 8 Jan 2026 09:40:09 +0000 Subject: [PATCH 16/18] #32: Add working tests --- lib/ants/io/load.py | 29 ++++++++-- .../tests/io/load/test_CallbackMetadata.py | 57 +++++++++++++++++++ 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 0753226..beed200 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -438,6 +438,7 @@ def __call__(self, cube, field, filename): filename = filename[0] metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) + print("m_file: ", metadata_files) if metadata_files != []: self._retrieve_metadata(metadata_files, cube) if self._user_callback is not None: @@ -457,15 +458,31 @@ def _retrieve_metadata(self, metadata_files, cube): The cube being loaded. """ + valid_metadata_names = ["license", "attribution", "restrictions"] + other_license= ["lisense", "licence", "lisence"] for metadata_file in metadata_files: - open_file = open(metadata_file, "r") - metadata = open_file.readlines() - open_file.close() file_name_splits = str(metadata_file).split(".") attribute_name = file_name_splits[-1] - cube.attributes[attribute_name] = metadata - with open("written_license.txt", "a") as file: - file.write("".join(metadata)) + if attribute_name in other_license: + warnings.warn(f"The attribute name {attribute_name} has been changed to " + "license, in line with ANTS working practices.", category=UserWarning) + attribute_name = "license" + if attribute_name not in valid_metadata_names: + warnings.warn(f"Attribute {attribute_name} is not a valid metadata file " + "name. Accepted metadata names are license, attribution " + "and restrictions.", category=UserWarning) + else: + print("cube attributes", cube.attributes) + if attribute_name in cube.attributes: + raise AttributeError(f"The {attribute_name} is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag.") + open_file = open(metadata_file, "r") + metadata = open_file.readlines() + open_file.close() + cube.attributes[attribute_name] = metadata + with open("written_license.txt", "a") as file: + file.write("".join(metadata)) def load_cube(*args, **kwargs): diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 50a3c5b..71b71dd 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -7,6 +7,7 @@ class directly. import iris import pytest import unittest.mock as mock +import warnings def test_metadata_files_added_to_attributes(tmp_path): @@ -81,3 +82,59 @@ def test_args_parsed_correctly_with_positional_args(tmp_path): iris.save(test_cube, str(temporary_cube_path)) ants.io.load.load_cube(temporary_cube_path, None, mock_callback) mock_callback.assert_called() + +def test_existing_metadata_error(tmp_path): + """Tests that an error is raised when a cube has existing metadata attributes.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + test_cube.attributes['license'] = 'an existing license' + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.license" + temporary_license_path.write_text(license_text, encoding="utf-8") + error_message="The license is already an attribute on the cube. To ignore metadata files, use the --ignore-metadata-files flag." + + with pytest.raises(AttributeError, match=error_message): + ants.io.load.load_cube(temporary_cube_path) + +def test_misspelt_license_warning(tmp_path): + """Tests that different spellings for license will raise a warning.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.lisense" + temporary_license_path.write_text(license_text, encoding="utf-8") + + warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + with pytest.raises(UserWarning, match=warning_message): + ants.io.load.load_cube(temporary_cube_path) + +def test_misspelt_license_added(tmp_path): + """Tests that a different spelling of license will add a license attribute.""" + license_text = "a license" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.lisense" + temporary_license_path.write_text(license_text, encoding="utf-8") + # ignore warning that will be raised + warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message=warning_message, category=UserWarning + ) + loaded_cube = ants.io.load.load_cube(temporary_cube_path) + assert loaded_cube.attributes['license'] == ['a license'] + +def test_invalid_metadata_name(tmp_path): + """Tests that an invalid metadata name will not be added as an attribute.""" + test_cube = ants.tests.stock.geodetic(shape=(2, 2)) + temporary_cube_path = tmp_path / "cube_attribute.nc" + iris.save(test_cube, str(temporary_cube_path)) + temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" + temporary_license_path.write_text(" ", encoding="utf-8") + + warning_message = "Attribute invalid-name is not a valid metadata file name. Accepted metadata names are license, attribution and restrictions." + with pytest.raises(UserWarning, match=warning_message): + ants.io.load.load_cube(temporary_cube_path) From ae8aab899e1e69ea5c648b5e253a6afa2b6047b2 Mon Sep 17 00:00:00 2001 From: Theo Geddes <108924122+mo-tgeddes@users.noreply.github.com> Date: Tue, 13 Jan 2026 08:46:54 +0000 Subject: [PATCH 17/18] #32: Run stylechecks --- lib/ants/cli/ancil_fill_n_merge.py | 2 +- lib/ants/io/load.py | 33 +++++++++------ .../tests/command_parse/test_integration.py | 4 +- .../tests/io/load/test_CallbackMetadata.py | 41 ++++++++++++++----- 4 files changed, 54 insertions(+), 26 deletions(-) diff --git a/lib/ants/cli/ancil_fill_n_merge.py b/lib/ants/cli/ancil_fill_n_merge.py index 3fee185..4776891 100755 --- a/lib/ants/cli/ancil_fill_n_merge.py +++ b/lib/ants/cli/ancil_fill_n_merge.py @@ -25,7 +25,7 @@ def load_data( land_fraction_threshold=None, begin=None, end=None, - ignore_metadata_files=None + ignore_metadata_files=None, ): """ Load the necessary data for performing a merge and fill operation. diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index beed200..6a4a801 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -56,8 +56,8 @@ """ import copy -import warnings import glob +import warnings from contextlib import contextmanager from functools import wraps @@ -374,9 +374,10 @@ def load_function(*args, **kwargs): ignore_metadata_files, type(ignore_metadata_files), ) - if ignore_metadata_files == False: + if not ignore_metadata_files: print("doing the thing") - # Do the handling for each way a user callback can be passed in through iris + # Do the handling for each way a user callback can be passed in through + # iris user_callback = None if len(args) == 3: user_callback = args[2] @@ -459,24 +460,32 @@ def _retrieve_metadata(self, metadata_files, cube): """ valid_metadata_names = ["license", "attribution", "restrictions"] - other_license= ["lisense", "licence", "lisence"] + other_license = ["lisense", "licence", "lisence"] for metadata_file in metadata_files: file_name_splits = str(metadata_file).split(".") attribute_name = file_name_splits[-1] if attribute_name in other_license: - warnings.warn(f"The attribute name {attribute_name} has been changed to " - "license, in line with ANTS working practices.", category=UserWarning) + warnings.warn( + f"The attribute name {attribute_name} has been changed to " + "license, in line with ANTS working practices.", + category=UserWarning, + ) attribute_name = "license" if attribute_name not in valid_metadata_names: - warnings.warn(f"Attribute {attribute_name} is not a valid metadata file " - "name. Accepted metadata names are license, attribution " - "and restrictions.", category=UserWarning) + warnings.warn( + f"Attribute {attribute_name} is not a valid metadata file " + "name. Accepted metadata names are license, attribution " + "and restrictions.", + category=UserWarning, + ) else: print("cube attributes", cube.attributes) if attribute_name in cube.attributes: - raise AttributeError(f"The {attribute_name} is already an attribute on the " - "cube. To ignore metadata files, use the " - "--ignore-metadata-files flag.") + raise AttributeError( + f"The {attribute_name} is already an attribute on the " + "cube. To ignore metadata files, use the " + "--ignore-metadata-files flag." + ) open_file = open(metadata_file, "r") metadata = open_file.readlines() open_file.close() diff --git a/lib/ants/tests/command_parse/test_integration.py b/lib/ants/tests/command_parse/test_integration.py index cd5f50c..50c11e0 100644 --- a/lib/ants/tests/command_parse/test_integration.py +++ b/lib/ants/tests/command_parse/test_integration.py @@ -213,7 +213,7 @@ def test_time_constraint_flags(self): begin=1990, end=1996, netcdf_only=False, - ignore_metadata_files=False + ignore_metadata_files=False, ) self.assertFalse(self.mock_config.called) self.assertEqual(args, target_args) @@ -282,7 +282,7 @@ def test_set_ignore_metadata_files_flag(self): "/path/to/lsm", "-o", "/path/to/output", - "--ignore-metadata-files" + "--ignore-metadata-files", ] with mock.patch("sys.argv", new=new): parser = AntsArgParser(target_lsm=True) diff --git a/lib/ants/tests/io/load/test_CallbackMetadata.py b/lib/ants/tests/io/load/test_CallbackMetadata.py index 71b71dd..5bc3d5d 100644 --- a/lib/ants/tests/io/load/test_CallbackMetadata.py +++ b/lib/ants/tests/io/load/test_CallbackMetadata.py @@ -1,13 +1,18 @@ +# (C) Crown Copyright, Met Office. All rights reserved. +# +# This file is part of ANTS and is released under the BSD 3-Clause license. +# See LICENSE.txt in the root of the repository for full licensing details. """ Includes tests for end-to-end functionality of CallbackMetadata as well as calling the class directly. """ +import unittest.mock as mock +import warnings + import ants.io.load import iris import pytest -import unittest.mock as mock -import warnings def test_metadata_files_added_to_attributes(tmp_path): @@ -83,20 +88,25 @@ def test_args_parsed_correctly_with_positional_args(tmp_path): ants.io.load.load_cube(temporary_cube_path, None, mock_callback) mock_callback.assert_called() + def test_existing_metadata_error(tmp_path): """Tests that an error is raised when a cube has existing metadata attributes.""" license_text = "a license" test_cube = ants.tests.stock.geodetic(shape=(2, 2)) - test_cube.attributes['license'] = 'an existing license' + test_cube.attributes["license"] = "an existing license" temporary_cube_path = tmp_path / "cube_attribute.nc" iris.save(test_cube, str(temporary_cube_path)) temporary_license_path = tmp_path / "cube_attribute.nc.license" temporary_license_path.write_text(license_text, encoding="utf-8") - error_message="The license is already an attribute on the cube. To ignore metadata files, use the --ignore-metadata-files flag." + error_message = ( + "The license is already an attribute on the cube. To ignore " + "metadata files, use the --ignore-metadata-files flag." + ) with pytest.raises(AttributeError, match=error_message): ants.io.load.load_cube(temporary_cube_path) + def test_misspelt_license_warning(tmp_path): """Tests that different spellings for license will raise a warning.""" license_text = "a license" @@ -106,10 +116,14 @@ def test_misspelt_license_warning(tmp_path): temporary_license_path = tmp_path / "cube_attribute.nc.lisense" temporary_license_path.write_text(license_text, encoding="utf-8") - warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + warning_message = ( + "The attribute name lisense has been changed to license, in " + "line with ANTS working practices." + ) with pytest.raises(UserWarning, match=warning_message): ants.io.load.load_cube(temporary_cube_path) + def test_misspelt_license_added(tmp_path): """Tests that a different spelling of license will add a license attribute.""" license_text = "a license" @@ -119,13 +133,15 @@ def test_misspelt_license_added(tmp_path): temporary_license_path = tmp_path / "cube_attribute.nc.lisense" temporary_license_path.write_text(license_text, encoding="utf-8") # ignore warning that will be raised - warning_message = "The attribute name lisense has been changed to license, in line with ANTS working practices." + warning_message = ( + "The attribute name lisense has been changed to license, in " + "line with ANTS working practices." + ) with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", message=warning_message, category=UserWarning - ) + warnings.filterwarnings("ignore", message=warning_message, category=UserWarning) loaded_cube = ants.io.load.load_cube(temporary_cube_path) - assert loaded_cube.attributes['license'] == ['a license'] + assert loaded_cube.attributes["license"] == ["a license"] + def test_invalid_metadata_name(tmp_path): """Tests that an invalid metadata name will not be added as an attribute.""" @@ -135,6 +151,9 @@ def test_invalid_metadata_name(tmp_path): temporary_license_path = tmp_path / "cube_attribute.nc.invalid-name" temporary_license_path.write_text(" ", encoding="utf-8") - warning_message = "Attribute invalid-name is not a valid metadata file name. Accepted metadata names are license, attribution and restrictions." + warning_message = ( + "Attribute invalid-name is not a valid metadata file name. " + "Accepted metadata names are license, attribution and restrictions." + ) with pytest.raises(UserWarning, match=warning_message): ants.io.load.load_cube(temporary_cube_path) From 2c00cd47cc1c891aa4ca921e9569712796492bff Mon Sep 17 00:00:00 2001 From: Theo Geddes Date: Thu, 29 Jan 2026 14:07:14 +0000 Subject: [PATCH 18/18] #32: remove print statements --- lib/ants/io/load.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/ants/io/load.py b/lib/ants/io/load.py index 6a4a801..b2a4107 100644 --- a/lib/ants/io/load.py +++ b/lib/ants/io/load.py @@ -361,21 +361,9 @@ def load_function(*args, **kwargs): FutureWarning, ) ignore_metadata_files = False - print("the func: ", func) - print("args: ", args) - print("kwargs: ", kwargs) - print("boo") - print("ignore_metadata_files" in kwargs) if "ignore_metadata_files" in kwargs: - print("its here!!!") ignore_metadata_files = kwargs.pop("ignore_metadata_files") - print( - "ignore metadata files: ", - ignore_metadata_files, - type(ignore_metadata_files), - ) if not ignore_metadata_files: - print("doing the thing") # Do the handling for each way a user callback can be passed in through # iris user_callback = None @@ -384,8 +372,6 @@ def load_function(*args, **kwargs): else: if "callback" in kwargs: user_callback = kwargs.pop("callback") - print("first args: ", args) - print("first kwargs: ", kwargs) args, kwargs = _add_callback( _CallbackMetadata(user_callback), *args, **kwargs ) @@ -434,16 +420,13 @@ def __call__(self, cube, field, filename): The method that runs when iris runs the callback. Collects the filenames and will run the user callback """ - print("callback has been added") if type(filename) is list: filename = filename[0] metadata_filenames = "".join([filename, ".*"]) metadata_files = glob.glob(metadata_filenames) - print("m_file: ", metadata_files) if metadata_files != []: self._retrieve_metadata(metadata_files, cube) if self._user_callback is not None: - print("should have added user callback") self._user_callback(cube, field, filename) def _retrieve_metadata(self, metadata_files, cube): @@ -479,7 +462,6 @@ def _retrieve_metadata(self, metadata_files, cube): category=UserWarning, ) else: - print("cube attributes", cube.attributes) if attribute_name in cube.attributes: raise AttributeError( f"The {attribute_name} is already an attribute on the "