32 add ancil load metadata

lib/ants/cli/ancil_2anc.py

@@ -37,16 +37,16 @@
 import iris
 
 
-def load_data(source):
+def load_data(source, ignore_metadata_files):
     with warnings.catch_warnings():
         warnings.filterwarnings(
             "ignore", "NetCDF default loading", iris._deprecation.IrisDeprecation
         )
-        cubes = ants.io.load.load(source)
+        cubes = ants.io.load.load(source, ignore_metadata_files=ignore_metadata_files)
     return cubes
 
 
-def main(source_path, output_path, grid_staggering, netcdf_only):
+def main(source_path, output_path, grid_staggering, netcdf_only, ignore_metadata_files):
     """
     Convert specified source file to an ancillary.
 
@@ -66,7 +66,7 @@ def main(source_path, output_path, grid_staggering, netcdf_only):
         written to an ancillary.
 
     """
-    source_cubes = load_data(source_path)
+    source_cubes = load_data(source_path, ignore_metadata_files)
     if grid_staggering is not None:
         for source_cube in source_cubes:
             source_cube.attributes["grid_staggering"] = grid_staggering
@@ -98,6 +98,7 @@ def cli_interface():
         args.output,
         args.grid_staggering,
         args.netcdf_only,
+        args.ignore_metadata_files,
     )
 
 

lib/ants/cli/ancil_fill_n_merge.py

@@ -25,6 +25,7 @@ def load_data(
     land_fraction_threshold=None,
     begin=None,
     end=None,
+    ignore_metadata_files=None,
 ):
     """
     Load the necessary data for performing a merge and fill operation.
@@ -56,12 +57,16 @@ def load_data(
         respectively.
 
     """
-    primary_cubes = ants.io.load.load(primary_source)
+    primary_cubes = ants.io.load.load(
+        primary_source, ignore_metadata_files=ignore_metadata_files
+    )
     if begin is not None:
         primary_cubes = create_time_constrained_cubes(primary_cubes, begin, end)
     alternate_cubes = None
     if alternate_source:
-        alternate_cubes = ants.io.load.load(alternate_source)
+        alternate_cubes = ants.io.load.load(
+            alternate_source, ignore_metadata_files=ignore_metadata_files
+        )
         if begin is not None:
             alternate_cubes = create_time_constrained_cubes(alternate_cubes, begin, end)
 
@@ -96,6 +101,7 @@ def main(
     end,
     netcdf_only,
     search_method,
+    ignore_metadata_files,
 ):
     """
     Perform merge and fill operation on the provided sources.
@@ -159,6 +165,7 @@ def main(
         land_fraction_threshold,
         begin,
         end,
+        ignore_metadata_files,
     )
 
     result = primary_cubes
@@ -251,6 +258,7 @@ def cli_interface():
         end=args.end,
         netcdf_only=args.netcdf_only,
         search_method=args.search_method,
+        ignore_metadata_files=args.ignore_metadata_files,
     )
 
 

lib/ants/cli/ancil_general_regrid.py

@@ -42,12 +42,17 @@ def load_data(
     land_fraction_threshold=None,
     begin=None,
     end=None,
+    ignore_metadata_files=None,
 ):
-    source_cubes = ants.io.load.load(source)
+    source_cubes = ants.io.load.load(
+        source, ignore_metadata_files=ignore_metadata_files
+    )
     if begin is not None:
         source_cubes = create_time_constrained_cubes(source_cubes, begin, end)
     if target_grid:
-        target_cube = ants.io.load.load_grid(target_grid)
+        target_cube = ants.io.load.load_grid(
+            target_grid, ignore_metadata_files=ignore_metadata_files
+        )
     else:
         target_cube = ants.io.load.load_landsea_mask(
             target_landseamask, land_fraction_threshold
@@ -76,6 +81,7 @@ def main(
     save_ukca,
     netcdf_only,
     search_method,
+    ignore_metadata_files,
 ):
     """
     General regrid application top level call function.
@@ -118,6 +124,9 @@ def main(
         provided source(s) consistent with the provided land sea mask.
         This should only be provided if a target land sea mask is also
         provided via target_lsm_path.
+    ignore_metadata_files : :obj:`bool`, optional
+        When set to True, files containing metadata will not be loaded alongside data
+        and added as attributes to the cube.
 
     Returns
     -------
@@ -132,6 +141,7 @@ def main(
         land_fraction_threshold,
         begin,
         end,
+        ignore_metadata_files,
     )
     if ants.utils.cube._is_ugrid(target_cube):
         raise ValueError(
@@ -210,6 +220,7 @@ def cli_interface():
         args.save_ukca,
         args.netcdf_only,
         args.search_method,
+        args.ignore_metadata_files,
     )
 
 

lib/ants/command_parse.py

@@ -33,6 +33,11 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False):
             sources <SOURCE1> <SOURCE2> ... <SOURCEN>     Source data path(s).
             --output <OUTPUT>, -o <OUTPUT>                Output filepath
             --ants-config <ANTS_CONFIG>                   Configuration path.
+            --ignore-metadata-files                       Stops the application from
+                                                          searching for metadata files
+                                                          with the same name as the
+                                                          source and loading them
+                                                          alongside the source.
 
         Additionally, there are standardised optional arguments and these are
         activated by passing the relevant keyword argument.  See 'Parameters'
@@ -171,6 +176,13 @@ def __init__(self, target_lsm=False, target_grid=False, time_constraints=False):
             help="Only write out a netCDF file.",
             required=False,
         )
+        self.add_argument(
+            "--ignore-metadata-files",
+            action="store_true",
+            help="Turn off the automatic loading of metadata files alongside source "
+            "files",
+            required=False,
+        )
         if time_constraints:
             self.add_argument(
                 "--begin",

lib/ants/io/load.py

@@ -56,6 +56,7 @@
 
 """
 import copy
+import glob
 import warnings
 from contextlib import contextmanager
 from functools import wraps
@@ -231,18 +232,22 @@ def load_landsea_mask(filename, land_threshold=None):
     """
     try:
         # Is it a landsea mask field?
-        lbm = ants.io.load.load_cube(filename, "land_binary_mask")
+        lbm = ants.io.load.load_cube(
+            filename, "land_binary_mask", ignore_metadata_files=True
+        )
         lbm = lbm.copy(lbm.data.astype("bool", copy=False))
     except iris.exceptions.ConstraintMismatchError:
         try:
             # Is it a land fraction field?
-            land_fraction = ants.io.load.load_cube(filename, "vegetation_area_fraction")
+            land_fraction = ants.io.load.load_cube(
+                filename, "vegetation_area_fraction", ignore_metadata_files=True
+            )
             lbm = land_fraction.copy(land_fraction.data > land_threshold)
             lbm.rename("land_binary_mask")
         except iris.exceptions.ConstraintMismatchError:
             # It looks like we are wanting to extract a landsea mask from some
             # other field.
-            cube = ants.io.load.load(filename)[0]
+            cube = ants.io.load.load(filename, ignore_metadata_files=True)[0]
             y = cube.coord(axis="y")
             x = cube.coord(axis="x")
             cube = cube.slices((y, x)).next()
@@ -355,6 +360,21 @@ def load_function(*args, **kwargs):
                 "iris.FUTURE.datum_support flag.",
                 FutureWarning,
             )
+            ignore_metadata_files = False
+            if "ignore_metadata_files" in kwargs:
+                ignore_metadata_files = kwargs.pop("ignore_metadata_files")
+            if not ignore_metadata_files:
+                # Do the handling for each way a user callback can be passed in through
+                # iris
+                user_callback = None
+                if len(args) == 3:
+                    user_callback = args[2]
+                else:
+                    if "callback" in kwargs:
+                        user_callback = kwargs.pop("callback")
+                args, kwargs = _add_callback(
+                    _CallbackMetadata(user_callback), *args, **kwargs
+                )
             # Use context manager to avoid permanently modifying iris behaviour.
             with ants_format_agent():
                 cubes = func(*args, **kwargs)
@@ -370,6 +390,92 @@ def load_function(*args, **kwargs):
     return load_function
 
 
+def _add_callback(callback, *args, **kwargs):
+    """
+    Adds both the ants callback and the user provided callback (if any) to the
+    load.
+    """
+    args = list(args)
+    if len(args) == 1:
+        kwargs["callback"] = callback
+    elif len(args) == 3:
+        args[2] = callback
+    args = tuple(args)
+    return args, kwargs
+
+
+class _CallbackMetadata(object):
+    """Callback for collecting metadata from sidecar files.
+
+    This callback will load additional metadata files with the naming convention:
+    filename.<metadata> and append the contents of those files to the cube
+    attributes.
+    """
+
+    def __init__(self, user_callback):
+        self._user_callback = user_callback
+
+    def __call__(self, cube, field, filename):
+        """
+        The method that runs when iris runs the callback. Collects the filenames and
+        will run the user callback
+        """
+        if type(filename) is list:
+            filename = filename[0]
+        metadata_filenames = "".join([filename, ".*"])
+        metadata_files = glob.glob(metadata_filenames)
+        if metadata_files != []:
+            self._retrieve_metadata(metadata_files, cube)
+        if self._user_callback is not None:
+            self._user_callback(cube, field, filename)
+
+    def _retrieve_metadata(self, metadata_files, cube):
+        """
+        Reads in the contents of each file and adds an attribute to the cube with the
+        name of the file.
+
+        Parameters
+        ----------
+        metadata_files: list
+            The list of filenames that contain metadata for the cube.
+        cube : :class:`iris.cube.Cube`
+            The cube being loaded.
+
+        """
+        valid_metadata_names = ["license", "attribution", "restrictions"]
+        other_license = ["lisense", "licence", "lisence"]
+        for metadata_file in metadata_files:
+            file_name_splits = str(metadata_file).split(".")
+            attribute_name = file_name_splits[-1]
+            if attribute_name in other_license:
+                warnings.warn(
+                    f"The attribute name {attribute_name} has been changed to "
+                    "license, in line with ANTS working practices.",
+                    category=UserWarning,
+                )
+                attribute_name = "license"
+            if attribute_name not in valid_metadata_names:
+                warnings.warn(
+                    f"Attribute {attribute_name} is not a valid metadata file "
+                    "name. Accepted metadata names are license, attribution "
+                    "and restrictions.",
+                    category=UserWarning,
+                )
+            else:
+                if attribute_name in cube.attributes:
+                    raise AttributeError(
+                        f"The {attribute_name} is already an attribute on the "
+                        "cube. To ignore metadata files, use the "
+                        "--ignore-metadata-files flag."
+                    )
+                open_file = open(metadata_file, "r")
+                metadata = open_file.readlines()
+                open_file.close()
+                cube.attributes[attribute_name] = metadata
+                with open("written_license.txt", "a") as file:
+                    file.write("".join(metadata))
+
+
 def load_cube(*args, **kwargs):
     """
     Loads a single cube.