diff options
Diffstat (limited to 'src/silx/utils')
32 files changed, 5281 insertions, 0 deletions
diff --git a/src/silx/utils/ExternalResources.py b/src/silx/utils/ExternalResources.py new file mode 100644 index 0000000..b79d6ff --- /dev/null +++ b/src/silx/utils/ExternalResources.py @@ -0,0 +1,321 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Helper to access to external resources. +""" + +__authors__ = ["Thomas Vincent", "J. Kieffer"] +__license__ = "MIT" +__date__ = "08/03/2019" + + +import os +import threading +import json +import logging +import tempfile +import unittest +import urllib.request +import urllib.error + +logger = logging.getLogger(__name__) + + +class ExternalResources(object): + """Utility class which allows to download test-data from www.silx.org + and manage the temporary data during the tests. + + """ + + def __init__(self, project, + url_base, + env_key=None, + timeout=60): + """Constructor of the class + + :param str project: name of the project, like "silx" + :param str url_base: base URL for the data, like "http://www.silx.org/pub" + :param str env_key: name of the environment variable which contains the + test_data directory, like "SILX_DATA". + If None (default), then the name of the + environment variable is built from the project argument: + "<PROJECT>_DATA". + The environment variable is optional: in case it is not set, + a directory in the temporary folder is used. + :param timeout: time in seconds before it breaks + """ + self.project = project + self._initialized = False + self.sem = threading.Semaphore() + + self.env_key = env_key or (self.project.upper() + "_TESTDATA") + self.url_base = url_base + self.all_data = set() + self.timeout = timeout + self._data_home = None + + @property + def data_home(self): + """Returns the data_home path and make sure it exists in the file + system.""" + if self._data_home is not None: + return self._data_home + + data_home = os.environ.get(self.env_key) + if data_home is None: + try: + import getpass + name = getpass.getuser() + except Exception: + if "getlogin" in dir(os): + name = os.getlogin() + elif "USER" in os.environ: + name = os.environ["USER"] + elif "USERNAME" in os.environ: + name = os.environ["USERNAME"] + else: + name = "uid" + str(os.getuid()) + + basename = "%s_testdata_%s" % (self.project, name) + data_home = os.path.join(tempfile.gettempdir(), basename) + if not os.path.exists(data_home): + os.makedirs(data_home) + self._data_home = data_home + return data_home + + def _initialize_data(self): + """Initialize for downloading test data""" + if not self._initialized: + with self.sem: + if not self._initialized: + self.testdata = os.path.join(self.data_home, "all_testdata.json") + if os.path.exists(self.testdata): + with open(self.testdata) as f: + self.all_data = set(json.load(f)) + self._initialized = True + + def clean_up(self): + pass + + def getfile(self, filename): + """Downloads the requested file from web-server available + at https://www.silx.org/pub/silx/ + + :param: relative name of the image. + :return: full path of the locally saved file. + """ + logger.debug("ExternalResources.getfile('%s')", filename) + + if not self._initialized: + self._initialize_data() + + fullfilename = os.path.abspath(os.path.join(self.data_home, filename)) + + if not os.path.isfile(fullfilename): + logger.debug("Trying to download image %s, timeout set to %ss", + filename, self.timeout) + dictProxies = {} + if "http_proxy" in os.environ: + dictProxies['http'] = os.environ["http_proxy"] + dictProxies['https'] = os.environ["http_proxy"] + if "https_proxy" in os.environ: + dictProxies['https'] = os.environ["https_proxy"] + if dictProxies: + proxy_handler = urllib.request.ProxyHandler(dictProxies) + opener = urllib.request.build_opener(proxy_handler).open + else: + opener = urllib.request.urlopen + + logger.debug("wget %s/%s", self.url_base, filename) + try: + data = opener("%s/%s" % (self.url_base, filename), + data=None, timeout=self.timeout).read() + logger.info("Image %s successfully downloaded.", filename) + except urllib.error.URLError: + raise unittest.SkipTest("network unreachable.") + + if not os.path.isdir(os.path.dirname(fullfilename)): + # Create sub-directory if needed + os.makedirs(os.path.dirname(fullfilename)) + + try: + with open(fullfilename, "wb") as outfile: + outfile.write(data) + except IOError: + raise IOError("unable to write downloaded \ + data to disk at %s" % self.data_home) + + if not os.path.isfile(fullfilename): + raise RuntimeError( + """Could not automatically download test images %s! + If you are behind a firewall, please set both environment variable http_proxy and https_proxy. + This even works under windows ! + Otherwise please try to download the images manually from + %s/%s""" % (filename, self.url_base, filename)) + + if filename not in self.all_data: + self.all_data.add(filename) + image_list = list(self.all_data) + image_list.sort() + try: + with open(self.testdata, "w") as fp: + json.dump(image_list, fp, indent=4) + except IOError: + logger.debug("Unable to save JSON list") + + return fullfilename + + def getdir(self, dirname): + """Downloads the requested tarball from the server + https://www.silx.org/pub/silx/ + and unzips it into the data directory + + :param: relative name of the image. + :return: list of files with their full path. + """ + lodn = dirname.lower() + if (lodn.endswith("tar") or lodn.endswith("tgz") or + lodn.endswith("tbz2") or lodn.endswith("tar.gz") or + lodn.endswith("tar.bz2")): + import tarfile + engine = tarfile.TarFile.open + elif lodn.endswith("zip"): + import zipfile + engine = zipfile.ZipFile + else: + raise RuntimeError("Unsupported archive format. Only tar and zip " + "are currently supported") + full_path = self.getfile(dirname) + with engine(full_path, mode="r") as fd: + output = os.path.join(self.data_home, dirname + "__content") + fd.extractall(output) + if lodn.endswith("zip"): + result = [os.path.join(output, i) for i in fd.namelist()] + else: + result = [os.path.join(output, i) for i in fd.getnames()] + return result + + def get_file_and_repack(self, filename): + """ + Download the requested file, decompress and repack it to bz2 and gz. + + :param str filename: name of the image. + :rtype: str + :return: full path of the locally saved file + """ + if not self._initialized: + self._initialize_data() + if filename not in self.all_data: + self.all_data.add(filename) + image_list = list(self.all_data) + image_list.sort() + try: + with open(self.testdata, "w") as fp: + json.dump(image_list, fp, indent=4) + except IOError: + logger.debug("Unable to save JSON list") + baseimage = os.path.basename(filename) + logger.info("UtilsTest.getimage('%s')" % baseimage) + + if not os.path.exists(self.data_home): + os.makedirs(self.data_home) + fullimagename = os.path.abspath(os.path.join(self.data_home, baseimage)) + + if baseimage.endswith(".bz2"): + bzip2name = baseimage + basename = baseimage[:-4] + gzipname = basename + ".gz" + elif baseimage.endswith(".gz"): + gzipname = baseimage + basename = baseimage[:-3] + bzip2name = basename + ".bz2" + else: + basename = baseimage + gzipname = baseimage + "gz2" + bzip2name = basename + ".bz2" + + fullimagename_gz = os.path.abspath(os.path.join(self.data_home, gzipname)) + fullimagename_raw = os.path.abspath(os.path.join(self.data_home, basename)) + fullimagename_bz2 = os.path.abspath(os.path.join(self.data_home, bzip2name)) + + # The files are recreated from the bz2 file + if not os.path.isfile(fullimagename_bz2): + self.getfile(bzip2name) + if not os.path.isfile(fullimagename_bz2): + raise RuntimeError( + """Could not automatically download test images %s! + If you are behind a firewall, please set the environment variable http_proxy. + Otherwise please try to download the images manually from + %s""" % (self.url_base, filename)) + + try: + import bz2 + except ImportError: + raise RuntimeError("bz2 library is needed to decompress data") + try: + import gzip + except ImportError: + gzip = None + + raw_file_exists = os.path.isfile(fullimagename_raw) + gz_file_exists = os.path.isfile(fullimagename_gz) + if not raw_file_exists or not gz_file_exists: + with open(fullimagename_bz2, "rb") as f: + data = f.read() + decompressed = bz2.decompress(data) + + if not raw_file_exists: + try: + with open(fullimagename_raw, "wb") as fullimage: + fullimage.write(decompressed) + except IOError: + raise IOError("unable to write decompressed \ + data to disk at %s" % self.data_home) + + if not gz_file_exists: + if gzip is None: + raise RuntimeError("gzip library is expected to recompress data") + try: + gzip.open(fullimagename_gz, "wb").write(decompressed) + except IOError: + raise IOError("unable to write gzipped \ + data to disk at %s" % self.data_home) + + return fullimagename + + def download_all(self, imgs=None): + """Download all data needed for the test/benchmarks + + :param imgs: list of files to download, by default all + :return: list of path with all files + """ + if not self._initialized: + self._initialize_data() + if not imgs: + imgs = self.all_data + res = [] + for fn in imgs: + logger.info("Downloading from silx.org: %s", fn) + res.append(self.getfile(fn)) + return res diff --git a/src/silx/utils/__init__.py b/src/silx/utils/__init__.py new file mode 100644 index 0000000..f803a5f --- /dev/null +++ b/src/silx/utils/__init__.py @@ -0,0 +1,28 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2018 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""This package contains a set of miscellaneous convenient features. + +See silx documentation: http://www.silx.org/doc/silx/latest/ +""" diff --git a/src/silx/utils/_have_openmp.pxd b/src/silx/utils/_have_openmp.pxd new file mode 100644 index 0000000..89a385c --- /dev/null +++ b/src/silx/utils/_have_openmp.pxd @@ -0,0 +1,49 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2018 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ + +""" +Store in a Cython module if it was compiled with OpenMP + +You have to patch the setup module like that: + +.. code-block:: python + + silx_include = os.path.join(top_path, "src", ""silx", "utils", "include") + config.add_extension('my_extension', + include_dirs=[silx_include], + ...) + +Then you can include it like that in your Cython module: + +.. code-block:: python + + include "../../utils/_have_openmp.pxi" + +""" + + +cdef extern from "silx_store_openmp.h": + int COMPILED_WITH_OPENMP +_COMPILED_WITH_OPENMP = COMPILED_WITH_OPENMP diff --git a/src/silx/utils/array_like.py b/src/silx/utils/array_like.py new file mode 100644 index 0000000..0cf4857 --- /dev/null +++ b/src/silx/utils/array_like.py @@ -0,0 +1,595 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Functions and classes for array-like objects, implementing common numpy +array features for datasets or nested sequences, while trying to avoid copying +data. + +Classes: + + - :class:`DatasetView`: Similar to a numpy view, to access + a h5py dataset as if it was transposed, without casting it into a + numpy array (this lets h5py handle reading the data from the + file into memory, as needed). + - :class:`ListOfImages`: Similar to a numpy view, to access + a list of 2D numpy arrays as if it was a 3D array (possibly transposed), + without casting it into a numpy array. + +Functions: + + - :func:`is_array` + - :func:`is_list_of_arrays` + - :func:`is_nested_sequence` + - :func:`get_shape` + - :func:`get_dtype` + - :func:`get_concatenated_dtype` + +""" + +from __future__ import absolute_import, print_function, division + +import sys + +import numpy +import numbers + +__authors__ = ["P. Knobel"] +__license__ = "MIT" +__date__ = "26/04/2017" + + +def is_array(obj): + """Return True if object implements necessary attributes to be + considered similar to a numpy array. + + Attributes needed are "shape", "dtype", "__getitem__" + and "__array__". + + :param obj: Array-like object (numpy array, h5py dataset...) + :return: boolean + """ + # add more required attribute if necessary + for attr in ("shape", "dtype", "__array__", "__getitem__"): + if not hasattr(obj, attr): + return False + return True + + +def is_list_of_arrays(obj): + """Return True if object is a sequence of numpy arrays, + e.g. a list of images as 2D arrays. + + :param obj: list of arrays + :return: boolean""" + # object must not be a numpy array + if is_array(obj): + return False + + # object must have a __len__ method + if not hasattr(obj, "__len__"): + return False + + # all elements in sequence must be arrays + for arr in obj: + if not is_array(arr): + return False + + return True + + +def is_nested_sequence(obj): + """Return True if object is a nested sequence. + + A simple 1D sequence is considered to be a nested sequence. + + Numpy arrays and h5py datasets are not considered to be nested sequences. + + To test if an object is a nested sequence in a more general sense, + including arrays and datasets, use:: + + is_nested_sequence(obj) or is_array(obj) + + :param obj: nested sequence (numpy array, h5py dataset...) + :return: boolean""" + # object must not be a numpy array + if is_array(obj): + return False + + if not hasattr(obj, "__len__"): + return False + + # obj must not be a list of (lists of) numpy arrays + subsequence = obj + while hasattr(subsequence, "__len__"): + if is_array(subsequence): + return False + # strings cause infinite loops + if isinstance(subsequence, (str, bytes)): + return True + subsequence = subsequence[0] + + # object has __len__ and is not an array + return True + + +def get_shape(array_like): + """Return shape of an array like object. + + In case the object is a nested sequence but not an array or dataset + (list of lists, tuples...), the size of each dimension is assumed to be + uniform, and is deduced from the length of the first sequence. + + :param array_like: Array like object: numpy array, hdf5 dataset, + multi-dimensional sequence + :return: Shape of array, as a tuple of integers + """ + if hasattr(array_like, "shape"): + return array_like.shape + + shape = [] + subsequence = array_like + while hasattr(subsequence, "__len__"): + shape.append(len(subsequence)) + # strings cause infinite loops + if isinstance(subsequence, (str, bytes)): + break + subsequence = subsequence[0] + + return tuple(shape) + + +def get_dtype(array_like): + """Return dtype of an array like object. + + In the case of a nested sequence, the type of the first value + is inspected. + + :param array_like: Array like object: numpy array, hdf5 dataset, + multi-dimensional nested sequence + :return: numpy dtype of object + """ + if hasattr(array_like, "dtype"): + return array_like.dtype + + subsequence = array_like + while hasattr(subsequence, "__len__"): + # strings cause infinite loops + if isinstance(subsequence, (str, bytes)): + break + subsequence = subsequence[0] + + return numpy.dtype(type(subsequence)) + + +def get_concatenated_dtype(arrays): + """Return dtype of array resulting of concatenation + of a list of arrays (without actually concatenating + them). + + :param arrays: list of numpy arrays + :return: resulting dtype after concatenating arrays + """ + dtypes = {a.dtype for a in arrays} + dummy = [] + for dt in dtypes: + dummy.append(numpy.zeros((1, 1), dtype=dt)) + return numpy.array(dummy).dtype + + +class ListOfImages(object): + """This class provides a way to access values and slices in a stack of + images stored as a list of 2D numpy arrays, without creating a 3D numpy + array first. + + A transposition can be specified, as a 3-tuple of dimensions in the wanted + order. For example, to transpose from ``xyz`` ``(0, 1, 2)`` into ``yzx``, + the transposition tuple is ``(1, 2, 0)`` + + All the 2D arrays in the list must have the same shape. + + The global dtype of the stack of images is the one that would be obtained + by casting the list of 2D arrays into a 3D numpy array. + + :param images: list of 2D numpy arrays, or :class:`ListOfImages` object + :param transposition: Tuple of dimension numbers in the wanted order + """ + def __init__(self, images, transposition=None): + """ + + """ + super(ListOfImages, self).__init__() + + # if images is a ListOfImages instance, get the underlying data + # as a list of 2D arrays + if isinstance(images, ListOfImages): + images = images.images + + # test stack of images is as expected + assert is_list_of_arrays(images), \ + "Image stack must be a list of arrays" + image0_shape = images[0].shape + for image in images: + assert image.ndim == 2, \ + "Images must be 2D numpy arrays" + assert image.shape == image0_shape, \ + "All images must have the same shape" + + self.images = images + """List of images""" + + self.shape = (len(images), ) + image0_shape + """Tuple of array dimensions""" + self.dtype = get_concatenated_dtype(images) + """Data-type of the global array""" + self.ndim = 3 + """Number of array dimensions""" + + self.size = len(images) * image0_shape[0] * image0_shape[1] + """Number of elements in the array.""" + + self.transposition = list(range(self.ndim)) + """List of dimension indices, in an order depending on the + specified transposition. By default this is simply + [0, ..., self.ndim], but it can be changed by specifying a different + ``transposition`` parameter at initialization. + + Use :meth:`transpose`, to create a new :class:`ListOfImages` + with a different :attr:`transposition`. + """ + + if transposition is not None: + assert len(transposition) == self.ndim + assert set(transposition) == set(list(range(self.ndim))), \ + "Transposition must be a sequence containing all dimensions" + self.transposition = transposition + self.__sort_shape() + + def __sort_shape(self): + """Sort shape in the order defined in :attr:`transposition` + """ + new_shape = tuple(self.shape[dim] for dim in self.transposition) + self.shape = new_shape + + def __sort_indices(self, indices): + """Return array indices sorted in the order needed + to access data in the original non-transposed images. + + :param indices: Tuple of ndim indices, in the order needed + to access the transposed view + :return: Sorted tuple of indices, to access original data + """ + assert len(indices) == self.ndim + sorted_indices = tuple(idx for (_, idx) in + sorted(zip(self.transposition, indices))) + return sorted_indices + + def __array__(self, dtype=None): + """Cast the images into a numpy array, and return it. + + If a transposition has been done on this images, return + a transposed view of a numpy array.""" + return numpy.transpose(numpy.array(self.images, dtype=dtype), + self.transposition) + + def __len__(self): + return self.shape[0] + + def transpose(self, transposition=None): + """Return a re-ordered (dimensions permutated) + :class:`ListOfImages`. + + The returned object refers to + the same images but with a different :attr:`transposition`. + + :param List[int] transposition: List/tuple of dimension numbers in the + wanted order. + If ``None`` (default), reverse the dimensions. + :return: new :class:`ListOfImages` object + """ + # by default, reverse the dimensions + if transposition is None: + transposition = list(reversed(self.transposition)) + + # If this ListOfImages is already transposed, sort new transposition + # relative to old transposition + elif list(self.transposition) != list(range(self.ndim)): + transposition = [self.transposition[i] for i in transposition] + + return ListOfImages(self.images, + transposition) + + @property + def T(self): + """ + Same as self.transpose() + + :return: DatasetView with dimensions reversed.""" + return self.transpose() + + def __getitem__(self, item): + """Handle a subset of numpy indexing with regards to the dimension + order as specified in :attr:`transposition` + + Following features are **not supported**: + + - fancy indexing using numpy arrays + - using ellipsis objects + + :param item: Index + :return: value or slice as a numpy array + """ + # 1-D slicing -> n-D slicing (n=1) + if not hasattr(item, "__len__"): + # first dimension index is given + item = [item] + # following dimensions are indexed with : (all elements) + item += [slice(None) for _i in range(self.ndim - 1)] + + # n-dimensional slicing + if len(item) != self.ndim: + raise IndexError( + "N-dim slicing requires a tuple of N indices/slices. " + + "Needed dimensions: %d" % self.ndim) + + # get list of indices sorted in the original images order + sorted_indices = self.__sort_indices(item) + list_idx, array_idx = sorted_indices[0], sorted_indices[1:] + + images_selection = self.images[list_idx] + + # now we must transpose the output data + output_dimensions = [] + frozen_dimensions = [] + for i, idx in enumerate(item): + # slices and sequences + if not isinstance(idx, numbers.Integral): + output_dimensions.append(self.transposition[i]) + # regular integer index + else: + # whenever a dimension is fixed (indexed by an integer) + # the number of output dimension is reduced + frozen_dimensions.append(self.transposition[i]) + + # decrement output dimensions that are above frozen dimensions + for frozen_dim in reversed(sorted(frozen_dimensions)): + for i, out_dim in enumerate(output_dimensions): + if out_dim > frozen_dim: + output_dimensions[i] -= 1 + + assert (len(output_dimensions) + len(frozen_dimensions)) == self.ndim + assert set(output_dimensions) == set(range(len(output_dimensions))) + + # single list elements selected + if isinstance(images_selection, numpy.ndarray): + return numpy.transpose(images_selection[array_idx], + axes=output_dimensions) + # muliple list elements selected + else: + # apply selection first + output_stack = [] + for img in images_selection: + output_stack.append(img[array_idx]) + # then cast into a numpy array, and transpose + return numpy.transpose(numpy.array(output_stack), + axes=output_dimensions) + + def min(self): + """ + :return: Global minimum value + """ + min_value = self.images[0].min() + if len(self.images) > 1: + for img in self.images[1:]: + min_value = min(min_value, img.min()) + return min_value + + def max(self): + """ + :return: Global maximum value + """ + max_value = self.images[0].max() + if len(self.images) > 1: + for img in self.images[1:]: + max_value = max(max_value, img.max()) + return max_value + + +class DatasetView(object): + """This class provides a way to transpose a dataset without + casting it into a numpy array. This way, the dataset in a file need not + necessarily be integrally read into memory to view it in a different + transposition. + + .. note:: + The performances depend a lot on the way the dataset was written + to file. Depending on the chunking strategy, reading a complete 2D slice + in an unfavorable direction may still require the entire dataset to + be read from disk. + + :param dataset: h5py dataset + :param transposition: List of dimensions sorted in the order of + transposition (relative to the original h5py dataset) + """ + def __init__(self, dataset, transposition=None): + """ + + """ + super(DatasetView, self).__init__() + self.dataset = dataset + """original dataset""" + + self.shape = dataset.shape + """Tuple of array dimensions""" + self.dtype = dataset.dtype + """Data-type of the array’s element""" + self.ndim = len(dataset.shape) + """Number of array dimensions""" + + size = 0 + if self.ndim: + size = 1 + for dimsize in self.shape: + size *= dimsize + self.size = size + """Number of elements in the array.""" + + self.transposition = list(range(self.ndim)) + """List of dimension indices, in an order depending on the + specified transposition. By default this is simply + [0, ..., self.ndim], but it can be changed by specifying a different + `transposition` parameter at initialization. + + Use :meth:`transpose`, to create a new :class:`DatasetView` + with a different :attr:`transposition`. + """ + + if transposition is not None: + assert len(transposition) == self.ndim + assert set(transposition) == set(list(range(self.ndim))), \ + "Transposition must be a list containing all dimensions" + self.transposition = transposition + self.__sort_shape() + + def __sort_shape(self): + """Sort shape in the order defined in :attr:`transposition` + """ + new_shape = tuple(self.shape[dim] for dim in self.transposition) + self.shape = new_shape + + def __sort_indices(self, indices): + """Return array indices sorted in the order needed + to access data in the original non-transposed dataset. + + :param indices: Tuple of ndim indices, in the order needed + to access the view + :return: Sorted tuple of indices, to access original data + """ + assert len(indices) == self.ndim + sorted_indices = tuple(idx for (_, idx) in + sorted(zip(self.transposition, indices))) + return sorted_indices + + def __getitem__(self, item): + """Handle fancy indexing with regards to the dimension order as + specified in :attr:`transposition` + + The supported fancy-indexing syntax is explained at + http://docs.h5py.org/en/latest/high/dataset.html#fancy-indexing. + + Additional restrictions exist if the data has been transposed: + + - numpy boolean array indexing is not supported + - ellipsis objects are not supported + + :param item: Index, possibly fancy index (must be supported by h5py) + :return: Sliced numpy array or numpy scalar + """ + # no transposition, let the original dataset handle indexing + if self.transposition == list(range(self.ndim)): + return self.dataset[item] + + # 1-D slicing: create a list of indices to switch to n-D slicing + if not hasattr(item, "__len__"): + # first dimension index (list index) is given + item = [item] + # following dimensions are indexed with slices representing all elements + item += [slice(None) for _i in range(self.ndim - 1)] + + # n-dimensional slicing + if len(item) != self.ndim: + raise IndexError( + "N-dim slicing requires a tuple of N indices/slices. " + + "Needed dimensions: %d" % self.ndim) + + # get list of indices sorted in the original dataset order + sorted_indices = self.__sort_indices(item) + + output_data_not_transposed = self.dataset[sorted_indices] + + # now we must transpose the output data + output_dimensions = [] + frozen_dimensions = [] + for i, idx in enumerate(item): + # slices and sequences + if not isinstance(idx, int): + output_dimensions.append(self.transposition[i]) + # regular integer index + else: + # whenever a dimension is fixed (indexed by an integer) + # the number of output dimension is reduced + frozen_dimensions.append(self.transposition[i]) + + # decrement output dimensions that are above frozen dimensions + for frozen_dim in reversed(sorted(frozen_dimensions)): + for i, out_dim in enumerate(output_dimensions): + if out_dim > frozen_dim: + output_dimensions[i] -= 1 + + assert (len(output_dimensions) + len(frozen_dimensions)) == self.ndim + assert set(output_dimensions) == set(range(len(output_dimensions))) + + return numpy.transpose(output_data_not_transposed, + axes=output_dimensions) + + def __array__(self, dtype=None): + """Cast the dataset into a numpy array, and return it. + + If a transposition has been done on this dataset, return + a transposed view of a numpy array.""" + return numpy.transpose(numpy.array(self.dataset, dtype=dtype), + self.transposition) + + def __len__(self): + return self.shape[0] + + def transpose(self, transposition=None): + """Return a re-ordered (dimensions permutated) + :class:`DatasetView`. + + The returned object refers to + the same dataset but with a different :attr:`transposition`. + + :param List[int] transposition: List of dimension numbers in the wanted order. + If ``None`` (default), reverse the dimensions. + :return: Transposed DatasetView + """ + # by default, reverse the dimensions + if transposition is None: + transposition = list(reversed(self.transposition)) + + # If this DatasetView is already transposed, sort new transposition + # relative to old transposition + elif list(self.transposition) != list(range(self.ndim)): + transposition = [self.transposition[i] for i in transposition] + + return DatasetView(self.dataset, + transposition) + + @property + def T(self): + """ + Same as self.transpose() + + :return: DatasetView with dimensions reversed.""" + return self.transpose() diff --git a/src/silx/utils/debug.py b/src/silx/utils/debug.py new file mode 100644 index 0000000..3d50fc9 --- /dev/null +++ b/src/silx/utils/debug.py @@ -0,0 +1,100 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ + + +import inspect +import types +import logging + + +debug_logger = logging.getLogger("silx.DEBUG") + +_indent = 0 + + +def log_method(func, class_name=None): + """Decorator to inject a warning log before an after any function/method. + + .. code-block:: python + + @log_method + def foo(): + return None + + :param callable func: The function to patch + :param str class_name: In case a method, provide the class name + """ + def wrapper(*args, **kwargs): + global _indent + + indent = " " * _indent + if class_name is not None: + name = "%s.%s" % (class_name, func.__name__) + else: + name = "%s" % func.__name__ + + debug_logger.warning("%s%s" % (indent, name)) + _indent += 1 + result = func(*args, **kwargs) + _indent -= 1 + debug_logger.warning("%sreturn (%s)" % (indent, name)) + return result + return wrapper + + +def log_all_methods(base_class): + """Decorator to inject a warning log before an after any method provided by + a class. + + .. code-block:: python + + @log_all_methods + class Foo(object): + + def a(self): + return None + + def b(self): + return self.a() + + Here is the output when calling the `b` method. + + .. code-block:: + + WARNING:silx.DEBUG:_Foobar.b + WARNING:silx.DEBUG: _Foobar.a + WARNING:silx.DEBUG: return (_Foobar.a) + WARNING:silx.DEBUG:return (_Foobar.b) + + :param class base_class: The class to patch + """ + methodTypes = (types.MethodType, types.FunctionType, types.BuiltinFunctionType, types.BuiltinMethodType) + for name, func in inspect.getmembers(base_class): + if isinstance(func, methodTypes): + if func.__name__ not in ["__subclasshook__", "__new__"]: + # patching __new__ in Python2 break the object, then we skip it + setattr(base_class, name, log_method(func, base_class.__name__)) + + return base_class diff --git a/src/silx/utils/deprecation.py b/src/silx/utils/deprecation.py new file mode 100644 index 0000000..7b19ee5 --- /dev/null +++ b/src/silx/utils/deprecation.py @@ -0,0 +1,123 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Bunch of useful decorators""" + +from __future__ import absolute_import, print_function, division + +__authors__ = ["Jerome Kieffer", "H. Payno", "P. Knobel"] +__license__ = "MIT" +__date__ = "26/02/2018" + +import sys +import logging +import functools +import traceback + +depreclog = logging.getLogger("silx.DEPRECATION") + +deprecache = set([]) + +FORCE = False +"""If true, deprecation using only_once are also generated. +It is needed for reproducible tests. +""" + +def deprecated(func=None, reason=None, replacement=None, since_version=None, only_once=True, skip_backtrace_count=1): + """ + Decorator that deprecates the use of a function + + :param str reason: Reason for deprecating this function + (e.g. "feature no longer provided", + :param str replacement: Name of replacement function (if the reason for + deprecating was to rename the function) + :param str since_version: First *silx* version for which the function was + deprecated (e.g. "0.5.0"). + :param bool only_once: If true, the deprecation warning will only be + generated one time. Default is true. + :param int skip_backtrace_count: Amount of last backtrace to ignore when + logging the backtrace + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + name = func.func_name if sys.version_info[0] < 3 else func.__name__ + + deprecated_warning(type_='Function', + name=name, + reason=reason, + replacement=replacement, + since_version=since_version, + only_once=only_once, + skip_backtrace_count=skip_backtrace_count) + return func(*args, **kwargs) + return wrapper + if func is not None: + return decorator(func) + return decorator + + +def deprecated_warning(type_, name, reason=None, replacement=None, + since_version=None, only_once=True, + skip_backtrace_count=0): + """ + Function to log a deprecation warning + + :param str type_: Nature of the object to be deprecated: + "Module", "Function", "Class" ... + :param name: Object name. + :param str reason: Reason for deprecating this function + (e.g. "feature no longer provided", + :param str replacement: Name of replacement function (if the reason for + deprecating was to rename the function) + :param str since_version: First *silx* version for which the function was + deprecated (e.g. "0.5.0"). + :param bool only_once: If true, the deprecation warning will only be + generated one time for each different call locations. Default is true. + :param int skip_backtrace_count: Amount of last backtrace to ignore when + logging the backtrace + """ + if not depreclog.isEnabledFor(logging.WARNING): + # Avoid computation when it is not logged + return + + msg = "%s %s is deprecated" + if since_version is not None: + msg += " since silx version %s" % since_version + msg += "." + if reason is not None: + msg += " Reason: %s." % reason + if replacement is not None: + msg += " Use '%s' instead." % replacement + msg += "\n%s" + limit = 2 + skip_backtrace_count + backtrace = "".join(traceback.format_stack(limit=limit)[0]) + backtrace = backtrace.rstrip() + if not FORCE and only_once: + data = (msg, type_, name, backtrace) + if data in deprecache: + return + else: + deprecache.add(data) + depreclog.warning(msg, type_, name, backtrace) diff --git a/src/silx/utils/enum.py b/src/silx/utils/enum.py new file mode 100644 index 0000000..fece575 --- /dev/null +++ b/src/silx/utils/enum.py @@ -0,0 +1,79 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2019 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""An :class:`.Enum` class with additional features.""" + +from __future__ import absolute_import + +__authors__ = ["T. Vincent"] +__license__ = "MIT" +__date__ = "29/04/2019" + + +import enum + + +class Enum(enum.Enum): + """Enum with additional class methods.""" + + @classmethod + def from_value(cls, value): + """Convert a value to corresponding Enum member + + :param value: The value to compare to Enum members + If it is already a member of Enum, it is returned directly. + :return: The corresponding enum member + :rtype: Enum + :raise ValueError: In case the conversion is not possible + """ + if isinstance(value, cls): + return value + for member in cls: + if value == member.value: + return member + raise ValueError("Cannot convert: %s" % value) + + @classmethod + def members(cls): + """Returns a tuple of all members. + + :rtype: Tuple[Enum] + """ + return tuple(member for member in cls) + + @classmethod + def names(cls): + """Returns a tuple of all member names. + + :rtype: Tuple[str] + """ + return tuple(member.name for member in cls) + + @classmethod + def values(cls): + """Returns a tuple of all member values. + + :rtype: Tuple + """ + return tuple(member.value for member in cls) diff --git a/src/silx/utils/exceptions.py b/src/silx/utils/exceptions.py new file mode 100644 index 0000000..addba89 --- /dev/null +++ b/src/silx/utils/exceptions.py @@ -0,0 +1,33 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Bunch of useful exceptions""" + +__authors__ = ["H. Payno"] +__license__ = "MIT" +__date__ = "17/01/2018" + + +class NotEditableError(Exception): + """Exception emitted when try to access to a non editable attribute""" diff --git a/src/silx/utils/files.py b/src/silx/utils/files.py new file mode 100644 index 0000000..1982c0d --- /dev/null +++ b/src/silx/utils/files.py @@ -0,0 +1,56 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2019 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Utils function relative to files +""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "19/09/2016" + +import os.path +import glob + +def expand_filenames(filenames): + """ + Takes a list of paths and expand it into a list of files. + + :param List[str] filenames: list of filenames or path with wildcards + :rtype: List[str] + :return: list of existing filenames or non-existing files + (which was provided as input) + """ + result = [] + for filename in filenames: + if os.path.exists(filename): + result.append(filename) + elif glob.has_magic(filename): + expanded_filenames = glob.glob(filename) + if expanded_filenames: + result += expanded_filenames + else: # Cannot expand, add as is + result.append(filename) + else: + result.append(filename) + return result diff --git a/src/silx/utils/html.py b/src/silx/utils/html.py new file mode 100644 index 0000000..9b39b95 --- /dev/null +++ b/src/silx/utils/html.py @@ -0,0 +1,37 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "19/09/2016" + +from .deprecation import deprecated_warning + +deprecated_warning(type_='module', + name=__name__, + replacement='html', + since_version='0.15.0') + +from html import escape # noqa diff --git a/src/silx/utils/include/silx_store_openmp.h b/src/silx/utils/include/silx_store_openmp.h new file mode 100644 index 0000000..f04f630 --- /dev/null +++ b/src/silx/utils/include/silx_store_openmp.h @@ -0,0 +1,10 @@ + + + +/** Flag the C module with a variable to know if it was compiled with OpenMP + */ +#ifdef _OPENMP +static const int COMPILED_WITH_OPENMP = 1; +#else +static const int COMPILED_WITH_OPENMP = 0; +#endif diff --git a/src/silx/utils/launcher.py b/src/silx/utils/launcher.py new file mode 100644 index 0000000..c46256a --- /dev/null +++ b/src/silx/utils/launcher.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2004-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""This module define silx application available throug the silx launcher. +""" + +__authors__ = ["P. Knobel", "V. Valls"] +__license__ = "MIT" +__date__ = "03/04/2017" + + +import sys +import importlib +import contextlib +import argparse +import logging + + +_logger = logging.getLogger(__name__) + + +class LauncherCommand(object): + """Description of a command""" + + def __init__(self, name, description=None, module_name=None, function=None): + """ + Constructor + + :param str name: Name of the command + :param str description: Description of the command + :param str module_name: Module name to execute + :param callable function: Python function to execute + """ + self.name = name + self.module_name = module_name + if description is None: + description = "A command" + self.description = description + self.function = function + + def get_module(self): + """Returns the python module to execute. If any. + + :rtype: module + """ + try: + module = importlib.import_module(self.module_name) + return module + except ImportError: + msg = "Error while reaching module '%s'" + _logger.error(msg, self.module_name, exc_info=True) + return None + + def get_function(self): + """Returns the main function to execute. + + :rtype: callable + """ + if self.function is not None: + return self.function + else: + module = self.get_module() + if module is None: + _logger.error("Impossible to load module name '%s'" % self.module_name) + return None + + # reach the 'main' function + if not hasattr(module, "main"): + raise TypeError("Module expect to have a 'main' function") + else: + main = getattr(module, "main") + return main + + @contextlib.contextmanager + def get_env(self, argv): + """Fix the environement before and after executing the command. + + :param list argv: The list of arguments (the first one is the name of + the application and command) + :rtype: int + """ + + # fix the context + old_argv = sys.argv + sys.argv = argv + + try: + yield + finally: + # clean up the context + sys.argv = old_argv + + def execute(self, argv): + """Execute the command. + + :param list[str] argv: The list of arguments (the first one is the + name of the application and command) + :rtype: int + :returns: The execution status + """ + with self.get_env(argv): + func = self.get_function() + if func is None: + _logger.error("Imposible to execute the command '%s'" % self.name) + return -1 + try: + status = func(argv) + except SystemExit as e: + # ArgumentParser likes to finish with a sys.exit + status = e.args[0] + return status + + +class Launcher(object): + """ + Manage launch of module. + + Provides an API to describe available commands and feature to display help + and execute the commands. + """ + + def __init__(self, + prog=None, + usage=None, + description=None, + epilog=None, + version=None): + """ + :param str prog: Name of the program. If it is not defined it uses the + first argument of `sys.argv` + :param str usage: Custom the string explaining the usage. Else it is + autogenerated. + :param str description: Description of the application displayed after the + usage. + :param str epilog: Custom the string displayed at the end of the help. + :param str version: Define the version of the application. + """ + if prog is None: + prog = sys.argv[0] + self.prog = prog + self.usage = usage + self.description = description + self.epilog = epilog + self.version = version + self._commands = {} + + help_command = LauncherCommand( + "help", + description="Show help of the following command", + function=self.execute_help) + self.add_command(command=help_command) + + def add_command(self, name=None, module_name=None, description=None, command=None): + """ + Add a command to the launcher. + + See also `LauncherCommand`. + + :param str name: Name of the command + :param str module_name: Module to execute + :param str description: Description of the command + :param LauncherCommand command: A `LauncherCommand` + """ + if command is not None: + assert(name is None and module_name is None and description is None) + else: + command = LauncherCommand( + name=name, + description=description, + module_name=module_name) + self._commands[command.name] = command + + def print_help(self): + """Print the help to stdout. + """ + usage = self.usage + if usage is None: + usage = "usage: {0.prog} [--version|--help] <command> [<args>]" + description = self.description + epilog = self.epilog + if epilog is None: + epilog = "See '{0.prog} help <command>' to read about a specific subcommand" + + print(usage.format(self)) + print("") + if description is not None: + print(description) + print("") + print("The {0.prog} commands are:".format(self)) + commands = sorted(self._commands.keys()) + for command in commands: + command = self._commands[command] + print(" {:10s} {:s}".format(command.name, command.description)) + print("") + print(epilog.format(self)) + + def execute_help(self, argv): + """Execute the help command. + + :param list[str] argv: The list of arguments (the first one is the + name of the application with the help command) + :rtype: int + :returns: The execution status + """ + description = "Display help information about %s" % self.prog + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + 'command', + default=None, + nargs=argparse.OPTIONAL, + help='Command in which aving help') + + try: + options = parser.parse_args(argv[1:]) + except SystemExit as e: + # ArgumentParser likes to finish with a sys.exit + return e.args[0] + + command_name = options.command + if command_name is None: + self.print_help() + return 0 + + if command_name not in self._commands: + print("Unknown command: %s", command_name) + self.print_help() + return -1 + + command = self._commands[command_name] + prog = "%s %s" % (self.prog, command.name) + return command.execute([prog, "--help"]) + + def execute(self, argv=None): + """Execute the launcher. + + :param list[str] argv: The list of arguments (the first one is the + name of the application) + :rtype: int + :returns: The execution status + """ + if argv is None: + argv = sys.argv + + if len(argv) <= 1: + self.print_help() + return 0 + + command_name = argv[1] + + if command_name == "--version": + print("%s version %s" % (self.prog, str(self.version))) + return 0 + + if command_name in ["--help", "-h"]: + # Special help command + if len(argv) == 2: + self.print_help() + return 0 + else: + command_name = argv[2] + command_argv = argv[2:] + ["--help"] + command_argv[0] = "%s %s" % (self.prog, command_argv[0]) + else: + command_argv = argv[1:] + command_argv[0] = "%s %s" % (self.prog, command_argv[0]) + + if command_name not in self._commands: + print("Unknown command: %s" % command_name) + self.print_help() + return -1 + + command = self._commands[command_name] + return command.execute(command_argv) diff --git a/src/silx/utils/number.py b/src/silx/utils/number.py new file mode 100755 index 0000000..f852a39 --- /dev/null +++ b/src/silx/utils/number.py @@ -0,0 +1,143 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2018 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Utilitary functions dealing with numbers. +""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "05/06/2018" + +import numpy +import re +import logging + + +_logger = logging.getLogger(__name__) + + +_biggest_float = None + +if hasattr(numpy, "longdouble"): + finfo = numpy.finfo(numpy.longdouble) + # The bit for sign is missing here + bits = finfo.nexp + finfo.nmant + if bits > 64: + _biggest_float = numpy.longdouble + # From bigger to smaller + _float_types = (numpy.longdouble, numpy.float64, numpy.float32, numpy.float16) +if _biggest_float is None: + _biggest_float = numpy.float64 + # From bigger to smaller + _float_types = (numpy.float64, numpy.float32, numpy.float16) + + +_parse_numeric_value = re.compile(r"^\s*[-+]?0*(\d+?)?(?:\.(\d+))?(?:[eE]([-+]?\d+))?\s*$") + + +def is_longdouble_64bits(): + """Returns true if the system uses floating-point 64bits for it's + long double type. + + .. note:: Comparing `numpy.longdouble` and `numpy.float64` on Windows is not + possible (or at least not will all the numpy version) + """ + return _biggest_float == numpy.float64 + + +def min_numerical_convertible_type(string, check_accuracy=True): + """ + Parse the string and try to return the smallest numerical type to use for + a safe conversion. It has some known issues: precission loss. + + :param str string: Representation of a float/integer with text + :param bool check_accuracy: If true, a warning is pushed on the logger + in case there is a loss of accuracy. + :raise ValueError: When the string is not a numerical value + :retrun: A numpy numerical type + """ + if string == "": + raise ValueError("Not a numerical value") + match = _parse_numeric_value.match(string) + if match is None: + raise ValueError("Not a numerical value") + number, decimal, exponent = match.groups() + + if decimal is None and exponent is None: + # It's an integer + # TODO: We could find the int type without converting the number + value = int(string) + return numpy.min_scalar_type(value).type + + # Try floating-point + try: + value = _biggest_float(string) + except ValueError: + raise ValueError("Not a numerical value") + + if number is None: + number = "" + if decimal is None: + decimal = "" + if exponent is None: + exponent = "0" + + nb_precision_digits = int(exponent) - len(decimal) - 1 + precision = _biggest_float(10) ** nb_precision_digits * 1.2 + previous_type = _biggest_float + for numpy_type in _float_types: + if numpy_type == _biggest_float: + # value was already casted using the bigger type + continue + reduced_value = numpy_type(value) + if not numpy.isfinite(reduced_value): + break + # numpy isclose(atol=is not accurate enough) + diff = value - reduced_value + # numpy 1.8.2 looks to do the substraction using float64... + # we lose precision here + diff = numpy.abs(diff) + if diff > precision: + break + previous_type = numpy_type + + # It's the smaller float type which fit with enougth precision + numpy_type = previous_type + + if check_accuracy and numpy_type == _biggest_float: + # Check the precision using the original string + expected = number + decimal + # This format the number without python convertion + try: + result = numpy.array2string(value, precision=len(number) + len(decimal), floatmode="fixed") + except TypeError: + # numpy 1.8.2 do not have floatmode argument + _logger.warning("Not able to check accuracy of the conversion of '%s' using %s", string, _biggest_float) + return numpy_type + + result = result.replace(".", "").replace("-", "") + if not result.startswith(expected): + _logger.warning("Not able to convert '%s' using %s without losing precision", string, _biggest_float) + + return numpy_type diff --git a/src/silx/utils/property.py b/src/silx/utils/property.py new file mode 100644 index 0000000..10d5d98 --- /dev/null +++ b/src/silx/utils/property.py @@ -0,0 +1,52 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Bunch of useful decorators""" + +from __future__ import absolute_import, print_function, division + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "22/02/2018" + + +class classproperty(property): + """ + Decorator to transform an object method into a class property. + + This code are equivalent, but the second one can be decorated with + deprecation warning for example. + + .. code-block:: python + + class Foo(object): + VALUE = 10 + + class Foo2(object): + @classproperty + def VALUE(self): + return 10 + """ + def __get__(self, cls, owner): + return classmethod(self.fget).__get__(None, owner)() diff --git a/src/silx/utils/proxy.py b/src/silx/utils/proxy.py new file mode 100644 index 0000000..d8821c2 --- /dev/null +++ b/src/silx/utils/proxy.py @@ -0,0 +1,208 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Module containing proxy objects""" + +from __future__ import absolute_import, print_function, division + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "02/10/2017" + + +import functools + + +class Proxy(object): + """Create a proxy of an object. + + Provides default methods and property using :meth:`__getattr__` and special + method by redefining them one by one. + Special methods are defined as properties, as a result if the `obj` method + is not defined, the property code fail and the special method will not be + visible. + """ + + __slots__ = ["__obj", "__weakref__"] + + def __init__(self, obj): + object.__setattr__(self, "_Proxy__obj", obj) + + __class__ = property(lambda self: self.__obj.__class__) + + def __getattr__(self, name): + return getattr(self.__obj, name) + + __setattr__ = property(lambda self: self.__obj.__setattr__) + __delattr__ = property(lambda self: self.__obj.__delattr__) + + # binary comparator methods + + __lt__ = property(lambda self: self.__obj.__lt__) + __le__ = property(lambda self: self.__obj.__le__) + __eq__ = property(lambda self: self.__obj.__eq__) + __ne__ = property(lambda self: self.__obj.__ne__) + __gt__ = property(lambda self: self.__obj.__gt__) + __ge__ = property(lambda self: self.__obj.__ge__) + + # binary numeric methods + + __add__ = property(lambda self: self.__obj.__add__) + __radd__ = property(lambda self: self.__obj.__radd__) + __iadd__ = property(lambda self: self.__obj.__iadd__) + __sub__ = property(lambda self: self.__obj.__sub__) + __rsub__ = property(lambda self: self.__obj.__rsub__) + __isub__ = property(lambda self: self.__obj.__isub__) + __mul__ = property(lambda self: self.__obj.__mul__) + __rmul__ = property(lambda self: self.__obj.__rmul__) + __imul__ = property(lambda self: self.__obj.__imul__) + + __truediv__ = property(lambda self: self.__obj.__truediv__) + __rtruediv__ = property(lambda self: self.__obj.__rtruediv__) + __itruediv__ = property(lambda self: self.__obj.__itruediv__) + __floordiv__ = property(lambda self: self.__obj.__floordiv__) + __rfloordiv__ = property(lambda self: self.__obj.__rfloordiv__) + __ifloordiv__ = property(lambda self: self.__obj.__ifloordiv__) + __mod__ = property(lambda self: self.__obj.__mod__) + __rmod__ = property(lambda self: self.__obj.__rmod__) + __imod__ = property(lambda self: self.__obj.__imod__) + __divmod__ = property(lambda self: self.__obj.__divmod__) + __rdivmod__ = property(lambda self: self.__obj.__rdivmod__) + __pow__ = property(lambda self: self.__obj.__pow__) + __rpow__ = property(lambda self: self.__obj.__rpow__) + __ipow__ = property(lambda self: self.__obj.__ipow__) + __lshift__ = property(lambda self: self.__obj.__lshift__) + __rlshift__ = property(lambda self: self.__obj.__rlshift__) + __ilshift__ = property(lambda self: self.__obj.__ilshift__) + __rshift__ = property(lambda self: self.__obj.__rshift__) + __rrshift__ = property(lambda self: self.__obj.__rrshift__) + __irshift__ = property(lambda self: self.__obj.__irshift__) + + # binary logical methods + + __and__ = property(lambda self: self.__obj.__and__) + __rand__ = property(lambda self: self.__obj.__rand__) + __iand__ = property(lambda self: self.__obj.__iand__) + __xor__ = property(lambda self: self.__obj.__xor__) + __rxor__ = property(lambda self: self.__obj.__rxor__) + __ixor__ = property(lambda self: self.__obj.__ixor__) + __or__ = property(lambda self: self.__obj.__or__) + __ror__ = property(lambda self: self.__obj.__ror__) + __ior__ = property(lambda self: self.__obj.__ior__) + + # unary methods + + __neg__ = property(lambda self: self.__obj.__neg__) + __pos__ = property(lambda self: self.__obj.__pos__) + __abs__ = property(lambda self: self.__obj.__abs__) + __invert__ = property(lambda self: self.__obj.__invert__) + __floor__ = property(lambda self: self.__obj.__floor__) + __ceil__ = property(lambda self: self.__obj.__ceil__) + __round__ = property(lambda self: self.__obj.__round__) + + # cast + + __repr__ = property(lambda self: self.__obj.__repr__) + __str__ = property(lambda self: self.__obj.__str__) + __complex__ = property(lambda self: self.__obj.__complex__) + __int__ = property(lambda self: self.__obj.__int__) + __float__ = property(lambda self: self.__obj.__float__) + __hash__ = property(lambda self: self.__obj.__hash__) + __bytes__ = property(lambda self: self.__obj.__bytes__) + __bool__ = property(lambda self: lambda: bool(self.__obj)) + __format__ = property(lambda self: self.__obj.__format__) + + # container + + __len__ = property(lambda self: self.__obj.__len__) + __length_hint__ = property(lambda self: self.__obj.__length_hint__) + __getitem__ = property(lambda self: self.__obj.__getitem__) + __missing__ = property(lambda self: self.__obj.__missing__) + __setitem__ = property(lambda self: self.__obj.__setitem__) + __delitem__ = property(lambda self: self.__obj.__delitem__) + __iter__ = property(lambda self: self.__obj.__iter__) + __reversed__ = property(lambda self: self.__obj.__reversed__) + __contains__ = property(lambda self: self.__obj.__contains__) + + # pickle + + __reduce__ = property(lambda self: self.__obj.__reduce__) + __reduce_ex__ = property(lambda self: self.__obj.__reduce_ex__) + + # async + + __await__ = property(lambda self: self.__obj.__await__) + __aiter__ = property(lambda self: self.__obj.__aiter__) + __anext__ = property(lambda self: self.__obj.__anext__) + __aenter__ = property(lambda self: self.__obj.__aenter__) + __aexit__ = property(lambda self: self.__obj.__aexit__) + + # other + + __index__ = property(lambda self: self.__obj.__index__) + + __next__ = property(lambda self: self.__obj.__next__) + + __enter__ = property(lambda self: self.__obj.__enter__) + __exit__ = property(lambda self: self.__obj.__exit__) + + __concat__ = property(lambda self: self.__obj.__concat__) + __iconcat__ = property(lambda self: self.__obj.__iconcat__) + + __call__ = property(lambda self: self.__obj.__call__) + + +def _docstring(dest, origin): + """Implementation of docstring decorator. + + It patches dest.__doc__. + """ + if not isinstance(dest, type) and isinstance(origin, type): + # func is not a class, but origin is, get the method with the same name + try: + origin = getattr(origin, dest.__name__) + except AttributeError: + raise ValueError( + "origin class has no %s method" % dest.__name__) + + dest.__doc__ = origin.__doc__ + return dest + + +def docstring(origin): + """Decorator to initialize the docstring from another source. + + This is useful to duplicate a docstring for inheritance and composition. + + If origin is a method or a function, it copies its docstring. + If origin is a class, the docstring is copied from the method + of that class which has the same name as the method/function + being decorated. + + :param origin: + The method, function or class from which to get the docstring + :raises ValueError: + If the origin class has not method n case the + """ + return functools.partial(_docstring, origin=origin) diff --git a/src/silx/utils/retry.py b/src/silx/utils/retry.py new file mode 100644 index 0000000..adc43bc --- /dev/null +++ b/src/silx/utils/retry.py @@ -0,0 +1,264 @@ +# coding: utf-8 +# /*########################################################################## +# Copyright (C) 2016-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ############################################################################*/ +""" +This module provides utility methods for retrying methods until they +no longer fail. +""" + +__authors__ = ["W. de Nolf"] +__license__ = "MIT" +__date__ = "05/02/2020" + + +import time +from functools import wraps +from contextlib import contextmanager +import multiprocessing +from queue import Empty + + +RETRY_PERIOD = 0.01 + + +class RetryTimeoutError(TimeoutError): + pass + + +class RetryError(RuntimeError): + pass + + +def _default_retry_on_error(e): + """ + :param BaseException e: + :returns bool: + """ + return isinstance(e, RetryError) + + +@contextmanager +def _handle_exception(options): + try: + yield + except BaseException as e: + retry_on_error = options.get("retry_on_error") + if retry_on_error is not None and retry_on_error(e): + options["exception"] = e + else: + raise + + +def _retry_loop(retry_timeout=None, retry_period=None, retry_on_error=None): + """Iterator which is endless or ends with an RetryTimeoutError. + It yields a dictionary which can be used to influence the loop. + + :param num retry_timeout: + :param num retry_period: sleep before retry + :param callable or None retry_on_error: checks whether an exception is + eligible for retry + """ + has_timeout = retry_timeout is not None + options = {"exception": None, "retry_on_error": retry_on_error} + if has_timeout: + t0 = time.time() + while True: + yield options + if retry_period is not None: + time.sleep(retry_period) + if has_timeout and (time.time() - t0) > retry_timeout: + raise RetryTimeoutError from options.get("exception") + + +def retry( + retry_timeout=None, retry_period=None, retry_on_error=_default_retry_on_error +): + """Decorator for a method that needs to be executed until it not longer + fails or until `retry_on_error` returns False. + + The decorator arguments can be overriden by using them when calling the + decorated method. + + :param num retry_timeout: + :param num retry_period: sleep before retry + :param callable or None retry_on_error: checks whether an exception is + eligible for retry + """ + + if retry_period is None: + retry_period = RETRY_PERIOD + + def decorator(method): + @wraps(method) + def wrapper(*args, **kw): + _retry_timeout = kw.pop("retry_timeout", retry_timeout) + _retry_period = kw.pop("retry_period", retry_period) + _retry_on_error = kw.pop("retry_on_error", retry_on_error) + for options in _retry_loop( + retry_timeout=_retry_timeout, + retry_period=_retry_period, + retry_on_error=_retry_on_error, + ): + with _handle_exception(options): + return method(*args, **kw) + + return wrapper + + return decorator + + +def retry_contextmanager( + retry_timeout=None, retry_period=None, retry_on_error=_default_retry_on_error +): + """Decorator to make a context manager from a method that needs to be + entered until it no longer fails or until `retry_on_error` returns False. + + The decorator arguments can be overriden by using them when calling the + decorated method. + + :param num retry_timeout: + :param num retry_period: sleep before retry + :param callable or None retry_on_error: checks whether an exception is + eligible for retry + """ + + if retry_period is None: + retry_period = RETRY_PERIOD + + def decorator(method): + @wraps(method) + def wrapper(*args, **kw): + _retry_timeout = kw.pop("retry_timeout", retry_timeout) + _retry_period = kw.pop("retry_period", retry_period) + _retry_on_error = kw.pop("retry_on_error", retry_on_error) + for options in _retry_loop( + retry_timeout=_retry_timeout, + retry_period=_retry_period, + retry_on_error=_retry_on_error, + ): + with _handle_exception(options): + gen = method(*args, **kw) + result = next(gen) + options["retry_on_error"] = None + yield result + try: + next(gen) + except StopIteration: + return + else: + raise RuntimeError(str(method) + " should only yield once") + + return contextmanager(wrapper) + + return decorator + + +def _subprocess_main(queue, method, retry_on_error, *args, **kw): + try: + result = method(*args, **kw) + except BaseException as e: + if retry_on_error(e): + # As the traceback gets lost, make sure the top-level + # exception is RetryError + e = RetryError(str(e)) + queue.put(e) + else: + queue.put(result) + + +def retry_in_subprocess( + retry_timeout=None, retry_period=None, retry_on_error=_default_retry_on_error +): + """Same as `retry` but it also retries segmentation faults. + + As subprocesses are spawned, you cannot use this decorator with the "@" syntax + because the decorated method needs to be an attribute of a module: + + .. code-block:: python + + def _method(*args, **kw): + ... + + method = retry_in_subprocess()(_method) + + :param num retry_timeout: + :param num retry_period: sleep before retry + :param callable or None retry_on_error: checks whether an exception is + eligible for retry + """ + + if retry_period is None: + retry_period = RETRY_PERIOD + + def decorator(method): + @wraps(method) + def wrapper(*args, **kw): + _retry_timeout = kw.pop("retry_timeout", retry_timeout) + _retry_period = kw.pop("retry_period", retry_period) + _retry_on_error = kw.pop("retry_on_error", retry_on_error) + + ctx = multiprocessing.get_context("spawn") + + def start_subprocess(): + queue = ctx.Queue(maxsize=1) + p = ctx.Process( + target=_subprocess_main, + args=(queue, method, retry_on_error) + args, + kwargs=kw, + ) + p.start() + return p, queue + + def stop_subprocess(p): + try: + p.kill() + except AttributeError: + p.terminate() + p.join() + + p, queue = start_subprocess() + try: + for options in _retry_loop( + retry_timeout=_retry_timeout, retry_on_error=_retry_on_error + ): + with _handle_exception(options): + if not p.is_alive(): + p, queue = start_subprocess() + try: + result = queue.get(block=True, timeout=_retry_period) + except Empty: + pass + except ValueError: + pass + else: + if isinstance(result, BaseException): + stop_subprocess(p) + raise result + else: + return result + finally: + stop_subprocess(p) + + return wrapper + + return decorator diff --git a/src/silx/utils/setup.py b/src/silx/utils/setup.py new file mode 100644 index 0000000..1f3e09a --- /dev/null +++ b/src/silx/utils/setup.py @@ -0,0 +1,43 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "24/08/2016" + + +from numpy.distutils.misc_util import Configuration + + +def configuration(parent_package='', top_path=None): + config = Configuration('utils', parent_package, top_path) + config.add_subpackage('test') + + return config + + +if __name__ == "__main__": + from numpy.distutils.core import setup + + setup(configuration=configuration) diff --git a/src/silx/utils/test/__init__.py b/src/silx/utils/test/__init__.py new file mode 100755 index 0000000..14fd940 --- /dev/null +++ b/src/silx/utils/test/__init__.py @@ -0,0 +1,24 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ diff --git a/src/silx/utils/test/test_array_like.py b/src/silx/utils/test/test_array_like.py new file mode 100644 index 0000000..a0b4b7b --- /dev/null +++ b/src/silx/utils/test/test_array_like.py @@ -0,0 +1,430 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for array_like module""" + +__authors__ = ["P. Knobel"] +__license__ = "MIT" +__date__ = "09/01/2017" + +import h5py +import numpy +import os +import tempfile +import unittest + +from ..array_like import DatasetView, ListOfImages +from ..array_like import get_dtype, get_concatenated_dtype, get_shape,\ + is_array, is_nested_sequence, is_list_of_arrays + + +class TestTransposedDatasetView(unittest.TestCase): + + def setUp(self): + # dataset attributes + self.ndim = 3 + self.original_shape = (5, 10, 20) + self.size = 1 + for dim in self.original_shape: + self.size *= dim + + self.volume = numpy.arange(self.size).reshape(self.original_shape) + + self.tempdir = tempfile.mkdtemp() + self.h5_fname = os.path.join(self.tempdir, "tempfile.h5") + with h5py.File(self.h5_fname, "w") as f: + f["volume"] = self.volume + + self.h5f = h5py.File(self.h5_fname, "r") + + self.all_permutations = [(0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), + (2, 0, 1), (2, 1, 0)] + + def tearDown(self): + self.h5f.close() + os.unlink(self.h5_fname) + os.rmdir(self.tempdir) + + def _testSize(self, obj): + """These assertions apply to all following test cases""" + self.assertEqual(obj.ndim, self.ndim) + self.assertEqual(obj.size, self.size) + size_from_shape = 1 + for dim in obj.shape: + size_from_shape *= dim + self.assertEqual(size_from_shape, self.size) + + for dim in self.original_shape: + self.assertIn(dim, obj.shape) + + def testNoTransposition(self): + """no transposition (transposition = (0, 1, 2))""" + a = DatasetView(self.h5f["volume"]) + + self.assertEqual(a.shape, self.original_shape) + self._testSize(a) + + # reversing the dimensions twice results in no change + rtrans = list(reversed(range(self.ndim))) + self.assertTrue(numpy.array_equal( + a, + a.transpose(rtrans).transpose(rtrans))) + + for i in range(a.shape[0]): + for j in range(a.shape[1]): + for k in range(a.shape[2]): + self.assertEqual(self.h5f["volume"][i, j, k], + a[i, j, k]) + + def _testTransposition(self, transposition): + """test transposed dataset + + :param tuple transposition: List of dimensions (0... n-1) sorted + in the desired order + """ + a = DatasetView(self.h5f["volume"], + transposition=transposition) + self._testSize(a) + + # sort shape of transposed object, to hopefully find the original shape + sorted_shape = tuple(dim_size for (_, dim_size) in + sorted(zip(transposition, a.shape))) + self.assertEqual(sorted_shape, self.original_shape) + + a_as_array = numpy.array(self.h5f["volume"]).transpose(transposition) + + # test the __array__ method + self.assertTrue(numpy.array_equal( + numpy.array(a), + a_as_array)) + + # test slicing + for selection in [(2, slice(None), slice(None)), + (slice(None), 1, slice(0, 8)), + (slice(0, 3), slice(None), 3), + (1, 3, slice(None)), + (slice(None), 2, 1), + (4, slice(1, 9, 2), 2)]: + self.assertIsInstance(a[selection], numpy.ndarray) + self.assertTrue(numpy.array_equal( + a[selection], + a_as_array[selection])) + + # test the DatasetView.__getitem__ for single values + # (step adjusted to test at least 3 indices in each dimension) + for i in range(0, a.shape[0], a.shape[0] // 3): + for j in range(0, a.shape[1], a.shape[1] // 3): + for k in range(0, a.shape[2], a.shape[2] // 3): + sorted_indices = tuple(idx for (_, idx) in + sorted(zip(transposition, [i, j, k]))) + viewed_value = a[i, j, k] + corresponding_original_value = self.h5f["volume"][sorted_indices] + self.assertEqual(viewed_value, + corresponding_original_value) + + # reversing the dimensions twice results in no change + rtrans = list(reversed(range(self.ndim))) + self.assertTrue(numpy.array_equal( + a, + a.transpose(rtrans).transpose(rtrans))) + + # test .T property + self.assertTrue(numpy.array_equal( + a.T, + a.transpose(rtrans))) + + def testTransposition012(self): + """transposition = (0, 1, 2) + (should be the same as testNoTransposition)""" + self._testTransposition((0, 1, 2)) + + def testTransposition021(self): + """transposition = (0, 2, 1)""" + self._testTransposition((0, 2, 1)) + + def testTransposition102(self): + """transposition = (1, 0, 2)""" + self._testTransposition((1, 0, 2)) + + def testTransposition120(self): + """transposition = (1, 2, 0)""" + self._testTransposition((1, 2, 0)) + + def testTransposition201(self): + """transposition = (2, 0, 1)""" + self._testTransposition((2, 0, 1)) + + def testTransposition210(self): + """transposition = (2, 1, 0)""" + self._testTransposition((2, 1, 0)) + + def testAllDoubleTranspositions(self): + for trans1 in self.all_permutations: + for trans2 in self.all_permutations: + self._testDoubleTransposition(trans1, trans2) + + def _testDoubleTransposition(self, transposition1, transposition2): + a = DatasetView(self.h5f["volume"], + transposition=transposition1).transpose(transposition2) + + b = self.volume.transpose(transposition1).transpose(transposition2) + + self.assertTrue(numpy.array_equal(a, b), + "failed with double transposition %s %s" % (transposition1, transposition2)) + + def test1DIndex(self): + a = DatasetView(self.h5f["volume"]) + self.assertTrue(numpy.array_equal(self.volume[1], + a[1])) + + b = DatasetView(self.h5f["volume"], transposition=(1, 0, 2)) + self.assertTrue(numpy.array_equal(self.volume[:, 1, :], + b[1])) + + +class TestTransposedListOfImages(unittest.TestCase): + def setUp(self): + # images attributes + self.ndim = 3 + self.original_shape = (5, 10, 20) + self.size = 1 + for dim in self.original_shape: + self.size *= dim + + volume = numpy.arange(self.size).reshape(self.original_shape) + + self.images = [] + for i in range(self.original_shape[0]): + self.images.append( + volume[i]) + + self.images_as_3D_array = numpy.array(self.images) + + self.all_permutations = [(0, 1, 2), (0, 2, 1), (1, 0, 2), (1, 2, 0), + (2, 0, 1), (2, 1, 0)] + + def tearDown(self): + pass + + def _testSize(self, obj): + """These assertions apply to all following test cases""" + self.assertEqual(obj.ndim, self.ndim) + self.assertEqual(obj.size, self.size) + size_from_shape = 1 + for dim in obj.shape: + size_from_shape *= dim + self.assertEqual(size_from_shape, self.size) + + for dim in self.original_shape: + self.assertIn(dim, obj.shape) + + def testNoTransposition(self): + """no transposition (transposition = (0, 1, 2))""" + a = ListOfImages(self.images) + + self.assertEqual(a.shape, self.original_shape) + self._testSize(a) + + for i in range(a.shape[0]): + for j in range(a.shape[1]): + for k in range(a.shape[2]): + self.assertEqual(self.images[i][j, k], + a[i, j, k]) + + # reversing the dimensions twice results in no change + rtrans = list(reversed(range(self.ndim))) + self.assertTrue(numpy.array_equal( + a, + a.transpose(rtrans).transpose(rtrans))) + + # test .T property + self.assertTrue(numpy.array_equal( + a.T, + a.transpose(rtrans))) + + def _testTransposition(self, transposition): + """test transposed dataset + + :param tuple transposition: List of dimensions (0... n-1) sorted + in the desired order + """ + a = ListOfImages(self.images, + transposition=transposition) + self._testSize(a) + + # sort shape of transposed object, to hopefully find the original shape + sorted_shape = tuple(dim_size for (_, dim_size) in + sorted(zip(transposition, a.shape))) + self.assertEqual(sorted_shape, self.original_shape) + + a_as_array = numpy.array(self.images).transpose(transposition) + + # test the DatasetView.__array__ method + self.assertTrue(numpy.array_equal( + numpy.array(a), + a_as_array)) + + # test slicing + for selection in [(2, slice(None), slice(None)), + (slice(None), 1, slice(0, 8)), + (slice(0, 3), slice(None), 3), + (1, 3, slice(None)), + (slice(None), 2, 1), + (4, slice(1, 9, 2), 2)]: + self.assertIsInstance(a[selection], numpy.ndarray) + self.assertTrue(numpy.array_equal( + a[selection], + a_as_array[selection])) + + # test the DatasetView.__getitem__ for single values + # (step adjusted to test at least 3 indices in each dimension) + for i in range(0, a.shape[0], a.shape[0] // 3): + for j in range(0, a.shape[1], a.shape[1] // 3): + for k in range(0, a.shape[2], a.shape[2] // 3): + viewed_value = a[i, j, k] + sorted_indices = tuple(idx for (_, idx) in + sorted(zip(transposition, [i, j, k]))) + corresponding_original_value = self.images[sorted_indices[0]][sorted_indices[1:]] + self.assertEqual(viewed_value, + corresponding_original_value) + + # reversing the dimensions twice results in no change + rtrans = list(reversed(range(self.ndim))) + self.assertTrue(numpy.array_equal( + a, + a.transpose(rtrans).transpose(rtrans))) + + # test .T property + self.assertTrue(numpy.array_equal( + a.T, + a.transpose(rtrans))) + + def _testDoubleTransposition(self, transposition1, transposition2): + a = ListOfImages(self.images, + transposition=transposition1).transpose(transposition2) + + b = self.images_as_3D_array.transpose(transposition1).transpose(transposition2) + + self.assertTrue(numpy.array_equal(a, b), + "failed with double transposition %s %s" % (transposition1, transposition2)) + + def testTransposition012(self): + """transposition = (0, 1, 2) + (should be the same as testNoTransposition)""" + self._testTransposition((0, 1, 2)) + + def testTransposition021(self): + """transposition = (0, 2, 1)""" + self._testTransposition((0, 2, 1)) + + def testTransposition102(self): + """transposition = (1, 0, 2)""" + self._testTransposition((1, 0, 2)) + + def testTransposition120(self): + """transposition = (1, 2, 0)""" + self._testTransposition((1, 2, 0)) + + def testTransposition201(self): + """transposition = (2, 0, 1)""" + self._testTransposition((2, 0, 1)) + + def testTransposition210(self): + """transposition = (2, 1, 0)""" + self._testTransposition((2, 1, 0)) + + def testAllDoubleTranspositions(self): + for trans1 in self.all_permutations: + for trans2 in self.all_permutations: + self._testDoubleTransposition(trans1, trans2) + + def test1DIndex(self): + a = ListOfImages(self.images) + self.assertTrue(numpy.array_equal(self.images[1], + a[1])) + + b = ListOfImages(self.images, transposition=(1, 0, 2)) + self.assertTrue(numpy.array_equal(self.images_as_3D_array[:, 1, :], + b[1])) + + +class TestFunctions(unittest.TestCase): + """Test functions to guess the dtype and shape of an array_like + object""" + def testListOfLists(self): + l = [[0, 1, 2], [2, 3, 4]] + self.assertEqual(get_dtype(l), + numpy.dtype(int)) + self.assertEqual(get_shape(l), + (2, 3)) + self.assertTrue(is_nested_sequence(l)) + self.assertFalse(is_array(l)) + self.assertFalse(is_list_of_arrays(l)) + + l = [[0., 1.], [2., 3.]] + self.assertEqual(get_dtype(l), + numpy.dtype(float)) + self.assertEqual(get_shape(l), + (2, 2)) + self.assertTrue(is_nested_sequence(l)) + self.assertFalse(is_array(l)) + self.assertFalse(is_list_of_arrays(l)) + + # concatenated dtype of int and float + l = [numpy.array([[0, 1, 2], [2, 3, 4]]), + numpy.array([[0., 1., 2.], [2., 3., 4.]])] + + self.assertEqual(get_concatenated_dtype(l), + numpy.array(l).dtype) + self.assertEqual(get_shape(l), + (2, 2, 3)) + self.assertFalse(is_nested_sequence(l)) + self.assertFalse(is_array(l)) + self.assertTrue(is_list_of_arrays(l)) + + def testNumpyArray(self): + a = numpy.array([[0, 1], [2, 3]]) + self.assertEqual(get_dtype(a), + a.dtype) + self.assertFalse(is_nested_sequence(a)) + self.assertTrue(is_array(a)) + self.assertFalse(is_list_of_arrays(a)) + + def testH5pyDataset(self): + a = numpy.array([[0, 1], [2, 3]]) + + tempdir = tempfile.mkdtemp() + h5_fname = os.path.join(tempdir, "tempfile.h5") + with h5py.File(h5_fname, "w") as h5f: + h5f["dataset"] = a + d = h5f["dataset"] + + self.assertEqual(get_dtype(d), + numpy.dtype(int)) + self.assertFalse(is_nested_sequence(d)) + self.assertTrue(is_array(d)) + self.assertFalse(is_list_of_arrays(d)) + + os.unlink(h5_fname) + os.rmdir(tempdir) diff --git a/src/silx/utils/test/test_debug.py b/src/silx/utils/test/test_debug.py new file mode 100644 index 0000000..09f4b01 --- /dev/null +++ b/src/silx/utils/test/test_debug.py @@ -0,0 +1,88 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for debug module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "27/02/2018" + + +import unittest +from silx.utils import debug +from silx.utils import testutils + + +@debug.log_all_methods +class _Foobar(object): + + def a(self): + return None + + def b(self): + return self.a() + + def random_args(self, *args, **kwargs): + return args, kwargs + + def named_args(self, a, b): + return a + 1, b + 1 + + +class TestDebug(unittest.TestCase): + """Tests for debug module.""" + + def logB(self): + """ + Can be used to check the log output using: + `./run_tests.py silx.utils.test.test_debug.TestDebug.logB -v` + """ + print() + test = _Foobar() + test.b() + + @testutils.validate_logging(debug.debug_logger.name, warning=2) + def testMethod(self): + test = _Foobar() + test.a() + + @testutils.validate_logging(debug.debug_logger.name, warning=4) + def testInterleavedMethod(self): + test = _Foobar() + test.b() + + @testutils.validate_logging(debug.debug_logger.name, warning=2) + def testNamedArgument(self): + # Arguments arre still provided to the patched method + test = _Foobar() + result = test.named_args(10, 11) + self.assertEqual(result, (11, 12)) + + @testutils.validate_logging(debug.debug_logger.name, warning=2) + def testRandomArguments(self): + # Arguments arre still provided to the patched method + test = _Foobar() + result = test.random_args("foo", 50, a=10, b=100) + self.assertEqual(result[0], ("foo", 50)) + self.assertEqual(result[1], {"a": 10, "b": 100}) diff --git a/src/silx/utils/test/test_deprecation.py b/src/silx/utils/test/test_deprecation.py new file mode 100644 index 0000000..d52cb26 --- /dev/null +++ b/src/silx/utils/test/test_deprecation.py @@ -0,0 +1,96 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for html module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "17/01/2018" + + +import unittest +from .. import deprecation +from silx.utils import testutils + + +class TestDeprecation(unittest.TestCase): + """Tests for deprecation module.""" + + @deprecation.deprecated + def deprecatedWithoutParam(self): + pass + + @deprecation.deprecated(reason="r", replacement="r", since_version="v") + def deprecatedWithParams(self): + pass + + @deprecation.deprecated(reason="r", replacement="r", since_version="v", only_once=True) + def deprecatedOnlyOnce(self): + pass + + @deprecation.deprecated(reason="r", replacement="r", since_version="v", only_once=False) + def deprecatedEveryTime(self): + pass + + @testutils.validate_logging(deprecation.depreclog.name, warning=1) + def testAnnotationWithoutParam(self): + self.deprecatedWithoutParam() + + @testutils.validate_logging(deprecation.depreclog.name, warning=1) + def testAnnotationWithParams(self): + self.deprecatedWithParams() + + @testutils.validate_logging(deprecation.depreclog.name, warning=3) + def testLoggedEveryTime(self): + """Logged everytime cause it is 3 different locations""" + self.deprecatedOnlyOnce() + self.deprecatedOnlyOnce() + self.deprecatedOnlyOnce() + + @testutils.validate_logging(deprecation.depreclog.name, warning=1) + def testLoggedSingleTime(self): + def log(): + self.deprecatedOnlyOnce() + log() + log() + log() + + @testutils.validate_logging(deprecation.depreclog.name, warning=3) + def testLoggedEveryTime2(self): + self.deprecatedEveryTime() + self.deprecatedEveryTime() + self.deprecatedEveryTime() + + @testutils.validate_logging(deprecation.depreclog.name, warning=1) + def testWarning(self): + deprecation.deprecated_warning(type_="t", name="n") + + def testBacktrace(self): + loggingValidator = testutils.LoggingValidator(deprecation.depreclog.name) + with loggingValidator: + self.deprecatedEveryTime() + message = loggingValidator.records[0].getMessage() + filename = __file__.replace(".pyc", ".py") + self.assertTrue(filename in message) + self.assertTrue("testBacktrace" in message) diff --git a/src/silx/utils/test/test_enum.py b/src/silx/utils/test/test_enum.py new file mode 100644 index 0000000..808304a --- /dev/null +++ b/src/silx/utils/test/test_enum.py @@ -0,0 +1,85 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2019 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests of Enum class with extra class methods""" + +from __future__ import absolute_import + +__authors__ = ["T. Vincent"] +__license__ = "MIT" +__date__ = "29/04/2019" + + +import sys +import unittest + +import enum +from silx.utils.enum import Enum + + +class TestEnum(unittest.TestCase): + """Tests for enum module.""" + + def test(self): + """Test with Enum""" + class Success(Enum): + A = 1 + B = 'B' + self._check_enum_content(Success) + + @unittest.skipIf(sys.version_info.major <= 2, 'Python3 only') + def test(self): + """Test Enum with member redefinition""" + with self.assertRaises(TypeError): + class Failure(Enum): + A = 1 + A = 'B' + + def test_unique(self): + """Test with enum.unique""" + with self.assertRaises(ValueError): + @enum.unique + class Failure(Enum): + A = 1 + B = 1 + + @enum.unique + class Success(Enum): + A = 1 + B = 'B' + self._check_enum_content(Success) + + def _check_enum_content(self, enum_): + """Check that the content of an enum is: <A: 1, B: 2>. + + :param Enum enum_: + """ + self.assertEqual(enum_.members(), (enum_.A, enum_.B)) + self.assertEqual(enum_.names(), ('A', 'B')) + self.assertEqual(enum_.values(), (1, 'B')) + + self.assertEqual(enum_.from_value(1), enum_.A) + self.assertEqual(enum_.from_value('B'), enum_.B) + with self.assertRaises(ValueError): + enum_.from_value(3) diff --git a/src/silx/utils/test/test_external_resources.py b/src/silx/utils/test/test_external_resources.py new file mode 100644 index 0000000..1fedda3 --- /dev/null +++ b/src/silx/utils/test/test_external_resources.py @@ -0,0 +1,89 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Test for resource files management.""" + +__authors__ = ["T. Vincent"] +__license__ = "MIT" +__date__ = "08/03/2019" + + +import os +import unittest +import shutil +import socket +import urllib.request +import urllib.error + +from silx.utils.ExternalResources import ExternalResources + + +def isSilxWebsiteAvailable(): + try: + urllib.request.urlopen('http://www.silx.org', timeout=1) + return True + except urllib.error.URLError: + return False + except socket.timeout: + # This exception is still received in Python 2.7 + return False + + +class TestExternalResources(unittest.TestCase): + """This is a test for the ExternalResources""" + + @classmethod + def setUpClass(cls): + if not isSilxWebsiteAvailable(): + raise unittest.SkipTest("Network or silx website not available") + + def setUp(self): + self.resources = ExternalResources("toto", "http://www.silx.org/pub/silx/") + + def tearDown(self): + if self.resources.data_home: + shutil.rmtree(self.resources.data_home) + self.resources = None + + def test_download(self): + "test the download from silx.org" + f = self.resources.getfile("lena.png") + self.assertTrue(os.path.exists(f)) + di = self.resources.getdir("source.tar.gz") + for fi in di: + self.assertTrue(os.path.exists(fi)) + + def test_download_all(self): + "test the download of all files from silx.org" + filename = self.resources.getfile("lena.png") + directory = "source.tar.gz" + filelist = self.resources.getdir(directory) + # download file and remove it to create a json mapping file + os.remove(filename) + directory_path = os.path.commonprefix(filelist) + # Make sure we will rmtree a dangerous path like "/" + self.assertIn(self.resources.data_home, directory_path) + shutil.rmtree(directory_path) + filelist = self.resources.download_all() + self.assertGreater(len(filelist), 1, "At least 2 items were downloaded") diff --git a/src/silx/utils/test/test_launcher.py b/src/silx/utils/test/test_launcher.py new file mode 100644 index 0000000..9e9024c --- /dev/null +++ b/src/silx/utils/test/test_launcher.py @@ -0,0 +1,191 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for html module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "17/01/2018" + + +import sys +import unittest +from silx.utils.testutils import ParametricTestCase +from .. import launcher + + +class CallbackMock(): + + def __init__(self, result=None): + self._execute_count = 0 + self._execute_argv = None + self._result = result + + def execute(self, argv): + self._execute_count = self._execute_count + 1 + self._execute_argv = argv + return self._result + + def __call__(self, argv): + return self.execute(argv) + + +class TestLauncherCommand(unittest.TestCase): + """Tests for launcher class.""" + + def testEnv(self): + command = launcher.LauncherCommand("foo") + old = sys.argv + params = ["foo", "bar"] + with command.get_env(params): + self.assertEqual(params, sys.argv) + self.assertEqual(sys.argv, old) + + def testEnvWhileException(self): + command = launcher.LauncherCommand("foo") + old = sys.argv + params = ["foo", "bar"] + try: + with command.get_env(params): + raise RuntimeError() + except RuntimeError: + pass + self.assertEqual(sys.argv, old) + + def testExecute(self): + params = ["foo", "bar"] + callback = CallbackMock(result=42) + command = launcher.LauncherCommand("foo", function=callback) + status = command.execute(params) + self.assertEqual(callback._execute_count, 1) + self.assertEqual(callback._execute_argv, params) + self.assertEqual(status, 42) + + +class TestModuleCommand(ParametricTestCase): + + def setUp(self): + module_name = "silx.utils.test.test_launcher_command" + command = launcher.LauncherCommand("foo", module_name=module_name) + self.command = command + + def testHelp(self): + status = self.command.execute(["--help"]) + self.assertEqual(status, 0) + + def testException(self): + try: + self.command.execute(["exception"]) + self.fail() + except RuntimeError: + pass + + def testCall(self): + status = self.command.execute([]) + self.assertEqual(status, 0) + + def testError(self): + status = self.command.execute(["error"]) + self.assertEqual(status, -1) + + +class TestLauncher(ParametricTestCase): + """Tests for launcher class.""" + + def testCallCommand(self): + callback = CallbackMock(result=42) + runner = launcher.Launcher(prog="prog") + command = launcher.LauncherCommand("foo", function=callback) + runner.add_command(command=command) + status = runner.execute(["prog", "foo", "param1", "param2"]) + self.assertEqual(status, 42) + self.assertEqual(callback._execute_argv, ["prog foo", "param1", "param2"]) + self.assertEqual(callback._execute_count, 1) + + def testAddCommand(self): + runner = launcher.Launcher(prog="prog") + module_name = "silx.utils.test.test_launcher_command" + runner.add_command("foo", module_name=module_name) + status = runner.execute(["prog", "foo"]) + self.assertEqual(status, 0) + + def testCallHelpOnCommand(self): + callback = CallbackMock(result=42) + runner = launcher.Launcher(prog="prog") + command = launcher.LauncherCommand("foo", function=callback) + runner.add_command(command=command) + status = runner.execute(["prog", "--help", "foo"]) + self.assertEqual(status, 42) + self.assertEqual(callback._execute_argv, ["prog foo", "--help"]) + self.assertEqual(callback._execute_count, 1) + + def testCallHelpOnCommand2(self): + callback = CallbackMock(result=42) + runner = launcher.Launcher(prog="prog") + command = launcher.LauncherCommand("foo", function=callback) + runner.add_command(command=command) + status = runner.execute(["prog", "help", "foo"]) + self.assertEqual(status, 42) + self.assertEqual(callback._execute_argv, ["prog foo", "--help"]) + self.assertEqual(callback._execute_count, 1) + + def testCallHelpOnUnknownCommand(self): + callback = CallbackMock(result=42) + runner = launcher.Launcher(prog="prog") + command = launcher.LauncherCommand("foo", function=callback) + runner.add_command(command=command) + status = runner.execute(["prog", "help", "foo2"]) + self.assertEqual(status, -1) + + def testNotAvailableCommand(self): + callback = CallbackMock(result=42) + runner = launcher.Launcher(prog="prog") + command = launcher.LauncherCommand("foo", function=callback) + runner.add_command(command=command) + status = runner.execute(["prog", "foo2", "param1", "param2"]) + self.assertEqual(status, -1) + self.assertEqual(callback._execute_count, 0) + + def testCallHelp(self): + callback = CallbackMock(result=42) + runner = launcher.Launcher(prog="prog") + command = launcher.LauncherCommand("foo", function=callback) + runner.add_command(command=command) + status = runner.execute(["prog", "help"]) + self.assertEqual(status, 0) + self.assertEqual(callback._execute_count, 0) + + def testCommonCommands(self): + runner = launcher.Launcher() + tests = [ + ["prog"], + ["prog", "--help"], + ["prog", "--version"], + ["prog", "help", "--help"], + ["prog", "help", "help"], + ] + for arguments in tests: + with self.subTest(args=tests): + status = runner.execute(arguments) + self.assertEqual(status, 0) diff --git a/src/silx/utils/test/test_launcher_command.py b/src/silx/utils/test/test_launcher_command.py new file mode 100644 index 0000000..ccf4601 --- /dev/null +++ b/src/silx/utils/test/test_launcher_command.py @@ -0,0 +1,47 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for html module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "03/04/2017" + + +import sys + + +def main(argv): + + if "--help" in argv: + # Common behaviour of ArgumentParser + sys.exit(0) + + if "exception" in argv: + raise RuntimeError("Simulated exception") + + if "error" in argv: + return -1 + + return 0 diff --git a/src/silx/utils/test/test_number.py b/src/silx/utils/test/test_number.py new file mode 100644 index 0000000..3eb8e91 --- /dev/null +++ b/src/silx/utils/test/test_number.py @@ -0,0 +1,175 @@ +# coding: utf-8 +# /*########################################################################## +# Copyright (C) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ############################################################################*/ +"""Tests for silx.uitls.number module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "05/06/2018" + +import logging +import numpy +import unittest +import pkg_resources +from silx.utils import number +from silx.utils import testutils + +_logger = logging.getLogger(__name__) + + +class TestConversionTypes(testutils.ParametricTestCase): + + def testEmptyFail(self): + self.assertRaises(ValueError, number.min_numerical_convertible_type, "") + + def testStringFail(self): + self.assertRaises(ValueError, number.min_numerical_convertible_type, "a") + + def testInteger(self): + dtype = number.min_numerical_convertible_type("1456") + self.assertTrue(numpy.issubdtype(dtype, numpy.unsignedinteger)) + + def testTrailledInteger(self): + dtype = number.min_numerical_convertible_type(" \t\n\r1456\t\n\r") + self.assertTrue(numpy.issubdtype(dtype, numpy.unsignedinteger)) + + def testPositiveInteger(self): + dtype = number.min_numerical_convertible_type("+1456") + self.assertTrue(numpy.issubdtype(dtype, numpy.unsignedinteger)) + + def testNegativeInteger(self): + dtype = number.min_numerical_convertible_type("-1456") + self.assertTrue(numpy.issubdtype(dtype, numpy.signedinteger)) + + def testIntegerExponential(self): + dtype = number.min_numerical_convertible_type("14e10") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testIntegerPositiveExponential(self): + dtype = number.min_numerical_convertible_type("14e+10") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testIntegerNegativeExponential(self): + dtype = number.min_numerical_convertible_type("14e-10") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testNumberDecimal(self): + dtype = number.min_numerical_convertible_type("14.5") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testPositiveNumberDecimal(self): + dtype = number.min_numerical_convertible_type("+14.5") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testNegativeNumberDecimal(self): + dtype = number.min_numerical_convertible_type("-14.5") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testDecimal(self): + dtype = number.min_numerical_convertible_type(".50") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testPositiveDecimal(self): + dtype = number.min_numerical_convertible_type("+.5") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testNegativeDecimal(self): + dtype = number.min_numerical_convertible_type("-.5") + self.assertTrue(numpy.issubdtype(dtype, numpy.floating)) + + def testMantissa16(self): + dtype = number.min_numerical_convertible_type("1.50") + self.assertEqual(dtype, numpy.float16) + + def testFloat32(self): + dtype = number.min_numerical_convertible_type("-23.172") + self.assertEqual(dtype, numpy.float32) + + def testMantissa32(self): + dtype = number.min_numerical_convertible_type("1400.50") + self.assertEqual(dtype, numpy.float32) + + def testMantissa64(self): + dtype = number.min_numerical_convertible_type("10000.000010") + self.assertEqual(dtype, numpy.float64) + + def testMantissa80(self): + self.skipIfFloat80NotSupported() + dtype = number.min_numerical_convertible_type("1000000000.00001013") + + if pkg_resources.parse_version(numpy.version.version) <= pkg_resources.parse_version("1.10.4"): + # numpy 1.8.2 -> Debian 8 + # Checking a float128 precision with numpy 1.8.2 using abs(diff) is not working. + # It looks like the difference is done using float64 (diff == 0.0) + expected = (numpy.longdouble, numpy.float64) + else: + expected = (numpy.longdouble, ) + self.assertIn(dtype, expected) + + def testExponent32(self): + dtype = number.min_numerical_convertible_type("14.0e30") + self.assertEqual(dtype, numpy.float32) + + def testExponent64(self): + dtype = number.min_numerical_convertible_type("14.0e300") + self.assertEqual(dtype, numpy.float64) + + def testExponent80(self): + self.skipIfFloat80NotSupported() + dtype = number.min_numerical_convertible_type("14.0e3000") + self.assertEqual(dtype, numpy.longdouble) + + def testFloat32ToString(self): + value = str(numpy.float32(numpy.pi)) + dtype = number.min_numerical_convertible_type(value) + self.assertIn(dtype, (numpy.float32, numpy.float64)) + + def skipIfFloat80NotSupported(self): + if number.is_longdouble_64bits(): + self.skipTest("float-80bits not supported") + + def testLosePrecisionUsingFloat80(self): + self.skipIfFloat80NotSupported() + if pkg_resources.parse_version(numpy.version.version) <= pkg_resources.parse_version("1.10.4"): + self.skipTest("numpy > 1.10.4 expected") + # value does not fit even in a 128 bits mantissa + value = "1.0340282366920938463463374607431768211456" + func = testutils.validate_logging(number._logger.name, warning=1) + func = func(number.min_numerical_convertible_type) + dtype = func(value) + self.assertIn(dtype, (numpy.longdouble, )) + + def testMillisecondEpochTime(self): + datetimes = ['1465803236.495412', + '1465803236.999362', + '1465803237.504311', + '1465803238.009261', + '1465803238.512211', + '1465803239.016160', + '1465803239.520110', + '1465803240.026059', + '1465803240.529009'] + for datetime in datetimes: + with self.subTest(datetime=datetime): + dtype = number.min_numerical_convertible_type(datetime) + self.assertEqual(dtype, numpy.float64) diff --git a/src/silx/utils/test/test_proxy.py b/src/silx/utils/test/test_proxy.py new file mode 100644 index 0000000..e165267 --- /dev/null +++ b/src/silx/utils/test/test_proxy.py @@ -0,0 +1,330 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2019 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for weakref module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "02/10/2017" + + +import unittest +import pickle +import numpy +from silx.utils.proxy import Proxy, docstring + + +class Thing(object): + + def __init__(self, value): + self.value = value + + def __getitem__(self, selection): + return selection + 1 + + def method(self, value): + return value + 2 + + +class InheritedProxy(Proxy): + """Inheriting the proxy allow to specialisze methods""" + + def __init__(self, obj, value): + Proxy.__init__(self, obj) + self.value = value + 2 + + def __getitem__(self, selection): + return selection + 3 + + def method(self, value): + return value + 4 + + +class TestProxy(unittest.TestCase): + """Test that the proxy behave as expected""" + + def text_init(self): + obj = Thing(10) + p = Proxy(obj) + self.assertTrue(isinstance(p, Thing)) + self.assertTrue(isinstance(p, Proxy)) + + # methods and properties + + def test_has_special_method(self): + obj = Thing(10) + p = Proxy(obj) + self.assertTrue(hasattr(p, "__getitem__")) + + def test_missing_special_method(self): + obj = Thing(10) + p = Proxy(obj) + self.assertFalse(hasattr(p, "__and__")) + + def test_method(self): + obj = Thing(10) + p = Proxy(obj) + self.assertEqual(p.method(10), obj.method(10)) + + def test_property(self): + obj = Thing(10) + p = Proxy(obj) + self.assertEqual(p.value, obj.value) + + # special functions + + def test_getitem(self): + obj = Thing(10) + p = Proxy(obj) + self.assertEqual(p[10], obj[10]) + + def test_setitem(self): + obj = numpy.array([10, 20, 30]) + p = Proxy(obj) + p[0] = 20 + self.assertEqual(obj[0], 20) + + def test_slice(self): + obj = numpy.arange(20) + p = Proxy(obj) + expected = obj[0:10:2] + result = p[0:10:2] + self.assertEqual(list(result), list(expected)) + + # binary comparator methods + + def test_lt(self): + obj = numpy.array([20]) + p = Proxy(obj) + expected = obj < obj + result = p < p + self.assertEqual(result, expected) + + # binary numeric methods + + def test_add(self): + obj = numpy.array([20]) + proxy = Proxy(obj) + expected = obj + obj + result = proxy + proxy + self.assertEqual(result, expected) + + def test_iadd(self): + expected = numpy.array([20]) + expected += 10 + obj = numpy.array([20]) + result = Proxy(obj) + result += 10 + self.assertEqual(result, expected) + + def test_radd(self): + obj = numpy.array([20]) + p = Proxy(obj) + expected = 10 + obj + result = 10 + p + self.assertEqual(result, expected) + + # binary logical methods + + def test_and(self): + obj = numpy.array([20]) + p = Proxy(obj) + expected = obj & obj + result = p & p + self.assertEqual(result, expected) + + def test_iand(self): + expected = numpy.array([20]) + expected &= 10 + obj = numpy.array([20]) + result = Proxy(obj) + result &= 10 + self.assertEqual(result, expected) + + def test_rand(self): + obj = numpy.array([20]) + p = Proxy(obj) + expected = 10 & obj + result = 10 & p + self.assertEqual(result, expected) + + # unary methods + + def test_neg(self): + obj = numpy.array([20]) + p = Proxy(obj) + expected = -obj + result = -p + self.assertEqual(result, expected) + + def test_round(self): + obj = 20.5 + p = Proxy(obj) + expected = round(obj) + result = round(p) + self.assertEqual(result, expected) + + # cast + + def test_bool(self): + obj = True + p = Proxy(obj) + if p: + pass + else: + self.fail() + + def test_str(self): + obj = Thing(10) + p = Proxy(obj) + expected = str(obj) + result = str(p) + self.assertEqual(result, expected) + + def test_repr(self): + obj = Thing(10) + p = Proxy(obj) + expected = repr(obj) + result = repr(p) + self.assertEqual(result, expected) + + def test_text_bool(self): + obj = "" + p = Proxy(obj) + if p: + self.fail() + else: + pass + + def test_text_str(self): + obj = "a" + p = Proxy(obj) + expected = str(obj) + result = str(p) + self.assertEqual(result, expected) + + def test_text_repr(self): + obj = "a" + p = Proxy(obj) + expected = repr(obj) + result = repr(p) + self.assertEqual(result, expected) + + def test_hash(self): + obj = [0, 1, 2] + p = Proxy(obj) + with self.assertRaises(TypeError): + hash(p) + obj = (0, 1, 2) + p = Proxy(obj) + hash(p) + + +class TestInheritedProxy(unittest.TestCase): + """Test that inheriting the Proxy class behave as expected""" + + # methods and properties + + def test_method(self): + obj = Thing(10) + p = InheritedProxy(obj, 11) + self.assertEqual(p.method(10), 11 + 3) + + def test_property(self): + obj = Thing(10) + p = InheritedProxy(obj, 11) + self.assertEqual(p.value, 11 + 2) + + # special functions + + def test_getitem(self): + obj = Thing(10) + p = InheritedProxy(obj, 11) + self.assertEqual(p[12], 12 + 3) + + +class TestPickle(unittest.TestCase): + + def test_dumps(self): + obj = Thing(10) + p = Proxy(obj) + expected = pickle.dumps(obj) + result = pickle.dumps(p) + self.assertEqual(result, expected) + + def test_loads(self): + obj = Thing(10) + p = Proxy(obj) + obj2 = pickle.loads(pickle.dumps(p)) + self.assertTrue(isinstance(obj2, Thing)) + self.assertFalse(isinstance(obj2, Proxy)) + self.assertEqual(obj.value, obj2.value) + + +class TestDocstring(unittest.TestCase): + """Test docstring decorator""" + + class Base(object): + def method(self): + """Docstring""" + pass + + def test_inheritance(self): + class Derived(TestDocstring.Base): + @docstring(TestDocstring.Base) + def method(self): + pass + + self.assertEqual(Derived.method.__doc__, + TestDocstring.Base.method.__doc__) + + def test_composition(self): + class Composed(object): + def __init__(self): + self._base = TestDocstring.Base() + + @docstring(TestDocstring.Base) + def method(self): + return self._base.method() + + @docstring(TestDocstring.Base.method) + def renamed(self): + return self._base.method() + + self.assertEqual(Composed.method.__doc__, + TestDocstring.Base.method.__doc__) + + self.assertEqual(Composed.renamed.__doc__, + TestDocstring.Base.method.__doc__) + + def test_function(self): + def f(): + """Docstring""" + pass + + @docstring(f) + def g(): + pass + + self.assertEqual(f.__doc__, g.__doc__) diff --git a/src/silx/utils/test/test_retry.py b/src/silx/utils/test/test_retry.py new file mode 100644 index 0000000..39bfdcf --- /dev/null +++ b/src/silx/utils/test/test_retry.py @@ -0,0 +1,169 @@ +# coding: utf-8 +# /*########################################################################## +# Copyright (C) 2016-2017 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ############################################################################*/ +"""Tests for retry utilities""" + +__authors__ = ["W. de Nolf"] +__license__ = "MIT" +__date__ = "05/02/2020" + + +import unittest +import os +import sys +import time +import tempfile + +from .. import retry + + +def _cause_segfault(): + import ctypes + + i = ctypes.c_char(b"a") + j = ctypes.pointer(i) + c = 0 + while True: + j[c] = b"a" + c += 1 + + +def _submain(filename, kwcheck=None, ncausefailure=0, faildelay=0): + assert filename + assert kwcheck + sys.stderr = open(os.devnull, "w") + + with open(filename, mode="r") as f: + failcounter = int(f.readline().strip()) + + if failcounter < ncausefailure: + time.sleep(faildelay) + failcounter += 1 + with open(filename, mode="w") as f: + f.write(str(failcounter)) + if failcounter % 2: + raise retry.RetryError + else: + _cause_segfault() + return True + + +_wsubmain = retry.retry_in_subprocess()(_submain) + + +class TestRetry(unittest.TestCase): + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.ctr_file = os.path.join(self.test_dir, "failcounter.txt") + + def tearDown(self): + if os.path.exists(self.ctr_file): + os.unlink(self.ctr_file) + os.rmdir(self.test_dir) + + def test_retry(self): + ncausefailure = 3 + faildelay = 0.1 + sufficient_timeout = ncausefailure * (faildelay + 10) + insufficient_timeout = ncausefailure * faildelay * 0.5 + + @retry.retry() + def method(check, kwcheck=None): + assert check + assert kwcheck + nonlocal failcounter + if failcounter < ncausefailure: + time.sleep(faildelay) + failcounter += 1 + raise retry.RetryError + return True + + failcounter = 0 + kw = { + "kwcheck": True, + "retry_timeout": sufficient_timeout, + } + self.assertTrue(method(True, **kw)) + + failcounter = 0 + kw = { + "kwcheck": True, + "retry_timeout": insufficient_timeout, + } + with self.assertRaises(retry.RetryTimeoutError): + method(True, **kw) + + def test_retry_contextmanager(self): + ncausefailure = 3 + faildelay = 0.1 + sufficient_timeout = ncausefailure * (faildelay + 10) + insufficient_timeout = ncausefailure * faildelay * 0.5 + + @retry.retry_contextmanager() + def context(check, kwcheck=None): + assert check + assert kwcheck + nonlocal failcounter + if failcounter < ncausefailure: + time.sleep(faildelay) + failcounter += 1 + raise retry.RetryError + yield True + + failcounter = 0 + kw = {"kwcheck": True, "retry_timeout": sufficient_timeout} + with context(True, **kw) as result: + self.assertTrue(result) + + failcounter = 0 + kw = {"kwcheck": True, "retry_timeout": insufficient_timeout} + with self.assertRaises(retry.RetryTimeoutError): + with context(True, **kw) as result: + pass + + def test_retry_in_subprocess(self): + ncausefailure = 3 + faildelay = 0.1 + sufficient_timeout = ncausefailure * (faildelay + 10) + insufficient_timeout = ncausefailure * faildelay * 0.5 + + kw = { + "ncausefailure": ncausefailure, + "faildelay": faildelay, + "kwcheck": True, + "retry_timeout": sufficient_timeout, + } + with open(self.ctr_file, mode="w") as f: + f.write("0") + self.assertTrue(_wsubmain(self.ctr_file, **kw)) + + kw = { + "ncausefailure": ncausefailure, + "faildelay": faildelay, + "kwcheck": True, + "retry_timeout": insufficient_timeout, + } + with open(self.ctr_file, mode="w") as f: + f.write("0") + with self.assertRaises(retry.RetryTimeoutError): + _wsubmain(self.ctr_file, **kw) diff --git a/src/silx/utils/test/test_testutils.py b/src/silx/utils/test/test_testutils.py new file mode 100755 index 0000000..4f07c4e --- /dev/null +++ b/src/silx/utils/test/test_testutils.py @@ -0,0 +1,94 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for testutils module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "18/11/2019" + + +import unittest +import logging +from .. import testutils + + +class TestLoggingValidator(unittest.TestCase): + """Tests for LoggingValidator""" + + def testRight(self): + logger = logging.getLogger(__name__ + "testRight") + listener = testutils.LoggingValidator(logger, error=1) + with listener: + logger.error("expected") + logger.info("ignored") + + def testCustomLevel(self): + logger = logging.getLogger(__name__ + "testCustomLevel") + listener = testutils.LoggingValidator(logger, error=1) + with listener: + logger.error("expected") + logger.log(666, "custom level have to be ignored") + + def testWrong(self): + logger = logging.getLogger(__name__ + "testWrong") + listener = testutils.LoggingValidator(logger, error=1) + with self.assertRaises(RuntimeError): + with listener: + logger.error("expected") + logger.error("not expected") + + def testManyErrors(self): + logger = logging.getLogger(__name__ + "testManyErrors") + listener = testutils.LoggingValidator(logger, error=1, warning=2) + with self.assertRaises(RuntimeError): + with listener: + pass + + def testCanBeChecked(self): + logger = logging.getLogger(__name__ + "testCanBreak") + listener = testutils.LoggingValidator(logger, error=1, warning=2) + with self.assertRaises(RuntimeError): + with listener: + logger.error("aaa") + logger.warning("aaa") + self.assertFalse(listener.can_be_checked()) + logger.error("aaa") + # Here we know that it's already wrong without a big cost + self.assertTrue(listener.can_be_checked()) + + def testWithAs(self): + logger = logging.getLogger(__name__ + "testCanBreak") + with testutils.LoggingValidator(logger) as listener: + logger.error("aaa") + self.assertIsNotNone(listener) + + def testErrorMessage(self): + logger = logging.getLogger(__name__ + "testCanBreak") + listener = testutils.LoggingValidator(logger, error=1, warning=2) + with self.assertRaisesRegex(RuntimeError, "aaabbb"): + with listener: + logger.error("aaa") + logger.warning("aaabbb") + logger.error("aaa") diff --git a/src/silx/utils/test/test_weakref.py b/src/silx/utils/test/test_weakref.py new file mode 100644 index 0000000..06f3adf --- /dev/null +++ b/src/silx/utils/test/test_weakref.py @@ -0,0 +1,315 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2019 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Tests for weakref module""" + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "15/09/2016" + + +import unittest +from .. import weakref + + +class Dummy(object): + """Dummy class to use it as geanie pig""" + def inc(self, a): + return a + 1 + + def __lt__(self, other): + return True + + +def dummy_inc(a): + """Dummy function to use it as geanie pig""" + return a + 1 + + +class TestWeakMethod(unittest.TestCase): + """Tests for weakref.WeakMethod""" + + def testMethod(self): + dummy = Dummy() + callable_ = weakref.WeakMethod(dummy.inc) + self.assertEqual(callable_()(10), 11) + + def testMethodWithDeadObject(self): + dummy = Dummy() + callable_ = weakref.WeakMethod(dummy.inc) + dummy = None + self.assertIsNone(callable_()) + + def testMethodWithDeadFunction(self): + dummy = Dummy() + dummy.inc2 = lambda self, a: a + 1 + callable_ = weakref.WeakMethod(dummy.inc2) + dummy.inc2 = None + self.assertIsNone(callable_()) + + def testFunction(self): + callable_ = weakref.WeakMethod(dummy_inc) + self.assertEqual(callable_()(10), 11) + + def testDeadFunction(self): + def inc(a): + return a + 1 + callable_ = weakref.WeakMethod(inc) + inc = None + self.assertIsNone(callable_()) + + def testLambda(self): + store = lambda a: a + 1 # noqa: E731 + callable_ = weakref.WeakMethod(store) + self.assertEqual(callable_()(10), 11) + + def testDeadLambda(self): + callable_ = weakref.WeakMethod(lambda a: a + 1) + self.assertIsNone(callable_()) + + def testCallbackOnDeadObject(self): + self.__count = 0 + + def callback(ref): + self.__count += 1 + self.assertIs(callable_, ref) + dummy = Dummy() + callable_ = weakref.WeakMethod(dummy.inc, callback) + dummy = None + self.assertEqual(self.__count, 1) + + def testCallbackOnDeadMethod(self): + self.__count = 0 + + def callback(ref): + self.__count += 1 + self.assertIs(callable_, ref) + dummy = Dummy() + dummy.inc2 = lambda self, a: a + 1 + callable_ = weakref.WeakMethod(dummy.inc2, callback) + dummy.inc2 = None + self.assertEqual(self.__count, 1) + + def testCallbackOnDeadFunction(self): + self.__count = 0 + + def callback(ref): + self.__count += 1 + self.assertIs(callable_, ref) + store = lambda a: a + 1 # noqa: E731 + callable_ = weakref.WeakMethod(store, callback) + store = None + self.assertEqual(self.__count, 1) + + def testEquals(self): + dummy = Dummy() + callable1 = weakref.WeakMethod(dummy.inc) + callable2 = weakref.WeakMethod(dummy.inc) + self.assertEqual(callable1, callable2) + + def testInSet(self): + callable_set = set([]) + dummy = Dummy() + callable_set.add(weakref.WeakMethod(dummy.inc)) + callable_ = weakref.WeakMethod(dummy.inc) + self.assertIn(callable_, callable_set) + + def testInDict(self): + callable_dict = {} + dummy = Dummy() + callable_dict[weakref.WeakMethod(dummy.inc)] = 10 + callable_ = weakref.WeakMethod(dummy.inc) + self.assertEqual(callable_dict.get(callable_), 10) + + +class TestWeakMethodProxy(unittest.TestCase): + + def testMethod(self): + dummy = Dummy() + callable_ = weakref.WeakMethodProxy(dummy.inc) + self.assertEqual(callable_(10), 11) + + def testMethodWithDeadObject(self): + dummy = Dummy() + method = weakref.WeakMethodProxy(dummy.inc) + dummy = None + self.assertRaises(ReferenceError, method, 9) + + +class TestWeakList(unittest.TestCase): + """Tests for weakref.WeakList""" + + def setUp(self): + self.list = weakref.WeakList() + self.object1 = Dummy() + self.object2 = Dummy() + self.list.append(self.object1) + self.list.append(self.object2) + + def testAppend(self): + obj = Dummy() + self.list.append(obj) + self.assertEqual(len(self.list), 3) + obj = None + self.assertEqual(len(self.list), 2) + + def testRemove(self): + self.list.remove(self.object1) + self.assertEqual(len(self.list), 1) + + def testPop(self): + obj = self.list.pop(0) + self.assertIs(obj, self.object1) + self.assertEqual(len(self.list), 1) + + def testGetItem(self): + self.assertIs(self.object1, self.list[0]) + + def testGetItemSlice(self): + objects = self.list[:] + self.assertEqual(len(objects), 2) + self.assertIs(self.object1, objects[0]) + self.assertIs(self.object2, objects[1]) + + def testIter(self): + obj_list = list(self.list) + self.assertEqual(len(obj_list), 2) + self.assertIs(self.object1, obj_list[0]) + + def testLen(self): + self.assertEqual(len(self.list), 2) + + def testSetItem(self): + obj = Dummy() + self.list[0] = obj + self.assertIsNot(self.object1, self.list[0]) + obj = None + self.assertEqual(len(self.list), 1) + + def testSetItemSlice(self): + obj = Dummy() + self.list[:] = [obj, obj] + self.assertEqual(len(self.list), 2) + self.assertIs(obj, self.list[0]) + self.assertIs(obj, self.list[1]) + obj = None + self.assertEqual(len(self.list), 0) + + def testDelItem(self): + del self.list[0] + self.assertEqual(len(self.list), 1) + self.assertIs(self.object2, self.list[0]) + + def testDelItemSlice(self): + del self.list[:] + self.assertEqual(len(self.list), 0) + + def testContains(self): + self.assertIn(self.object1, self.list) + + def testAdd(self): + others = [Dummy()] + l = self.list + others + self.assertIs(l[0], self.object1) + self.assertEqual(len(l), 3) + others = None + self.assertEqual(len(l), 2) + + def testExtend(self): + others = [Dummy()] + self.list.extend(others) + self.assertIs(self.list[0], self.object1) + self.assertEqual(len(self.list), 3) + others = None + self.assertEqual(len(self.list), 2) + + def testIadd(self): + others = [Dummy()] + self.list += others + self.assertIs(self.list[0], self.object1) + self.assertEqual(len(self.list), 3) + others = None + self.assertEqual(len(self.list), 2) + + def testMul(self): + l = self.list * 2 + self.assertIs(l[0], self.object1) + self.assertEqual(len(l), 4) + self.object1 = None + self.assertEqual(len(l), 2) + self.assertIs(l[0], self.object2) + self.assertIs(l[1], self.object2) + + def testImul(self): + self.list *= 2 + self.assertIs(self.list[0], self.object1) + self.assertEqual(len(self.list), 4) + self.object1 = None + self.assertEqual(len(self.list), 2) + self.assertIs(self.list[0], self.object2) + self.assertIs(self.list[1], self.object2) + + def testCount(self): + self.list.append(self.object2) + self.assertEqual(self.list.count(self.object1), 1) + self.assertEqual(self.list.count(self.object2), 2) + + def testIndex(self): + self.assertEqual(self.list.index(self.object1), 0) + self.assertEqual(self.list.index(self.object2), 1) + + def testInsert(self): + obj = Dummy() + self.list.insert(1, obj) + self.assertEqual(len(self.list), 3) + self.assertIs(self.list[1], obj) + obj = None + self.assertEqual(len(self.list), 2) + + def testReverse(self): + self.list.reverse() + self.assertEqual(len(self.list), 2) + self.assertIs(self.list[0], self.object2) + self.assertIs(self.list[1], self.object1) + + def testReverted(self): + new_list = reversed(self.list) + self.assertEqual(len(new_list), 2) + self.assertIs(self.list[1], self.object2) + self.assertIs(self.list[0], self.object1) + self.assertIs(new_list[0], self.object2) + self.assertIs(new_list[1], self.object1) + self.object1 = None + self.assertEqual(len(new_list), 1) + + def testStr(self): + self.assertNotEqual(self.list.__str__(), "[]") + + def testRepr(self): + self.assertNotEqual(self.list.__repr__(), "[]") + + def testSort(self): + # only a coverage + self.list.sort() + self.assertEqual(len(self.list), 2) diff --git a/src/silx/utils/testutils.py b/src/silx/utils/testutils.py new file mode 100755 index 0000000..4177e33 --- /dev/null +++ b/src/silx/utils/testutils.py @@ -0,0 +1,351 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2021 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Utilities for writing tests. + +- :class:`ParametricTestCase` provides a :meth:`TestCase.subTest` replacement + for Python < 3.4 +- :class:`LoggingValidator` with context or the :func:`validate_logging` + decorator enables testing the number of logging messages of different levels. +""" + +__authors__ = ["T. Vincent"] +__license__ = "MIT" +__date__ = "26/01/2018" + + +import contextlib +import functools +import logging +import sys +import unittest +from . import deprecation + + +_logger = logging.getLogger(__name__) + + +if sys.hexversion >= 0x030400F0: # Python >= 3.4 + class ParametricTestCase(unittest.TestCase): + pass +else: + class ParametricTestCase(unittest.TestCase): + """TestCase with subTest support for Python < 3.4. + + Add subTest method to support parametric tests. + API is the same, but behavior differs: + If a subTest fails, the following ones are not run. + """ + + _subtest_msg = None # Class attribute to provide a default value + + @contextlib.contextmanager + def subTest(self, msg=None, **params): + """Use as unittest.TestCase.subTest method in Python >= 3.4.""" + # Format arguments as: '[msg] (key=value, ...)' + param_str = ', '.join(['%s=%s' % (k, v) for k, v in params.items()]) + self._subtest_msg = '[%s] (%s)' % (msg or '', param_str) + yield + self._subtest_msg = None + + def shortDescription(self): + short_desc = super(ParametricTestCase, self).shortDescription() + if self._subtest_msg is not None: + # Append subTest message to shortDescription + short_desc = ' '.join( + [msg for msg in (short_desc, self._subtest_msg) if msg]) + + return short_desc if short_desc else None + + +def parameterize(test_case_class, *args, **kwargs): + """Create a suite containing all tests taken from the given + subclass, passing them the parameters. + + .. code-block:: python + + class TestParameterizedCase(unittest.TestCase): + def __init__(self, methodName='runTest', foo=None): + unittest.TestCase.__init__(self, methodName) + self.foo = foo + + def suite(): + testSuite = unittest.TestSuite() + testSuite.addTest(parameterize(TestParameterizedCase, foo=10)) + testSuite.addTest(parameterize(TestParameterizedCase, foo=50)) + return testSuite + """ + test_loader = unittest.TestLoader() + test_names = test_loader.getTestCaseNames(test_case_class) + suite = unittest.TestSuite() + for name in test_names: + suite.addTest(test_case_class(name, *args, **kwargs)) + return suite + + +class LoggingRuntimeError(RuntimeError): + """Raised when the `LoggingValidator` fails""" + + def __init__(self, msg, records): + super(LoggingRuntimeError, self).__init__(msg) + self.records = records + + def __str__(self): + return super(LoggingRuntimeError, self).__str__() + " -> " + str(self.records) + + +class LoggingValidator(logging.Handler): + """Context checking the number of logging messages from a specified Logger. + + It disables propagation of logging message while running. + + This is meant to be used as a with statement, for example: + + >>> with LoggingValidator(logger, error=2, warning=0): + >>> pass # Run tests here expecting 2 ERROR and no WARNING from logger + ... + + :param logger: Name or instance of the logger to test. + (Default: root logger) + :type logger: str or :class:`logging.Logger` + :param int critical: Expected number of CRITICAL messages. + Default: Do not check. + :param int error: Expected number of ERROR messages. + Default: Do not check. + :param int warning: Expected number of WARNING messages. + Default: Do not check. + :param int info: Expected number of INFO messages. + Default: Do not check. + :param int debug: Expected number of DEBUG messages. + Default: Do not check. + :param int notset: Expected number of NOTSET messages. + Default: Do not check. + :raises RuntimeError: If the message counts are the expected ones. + """ + + def __init__(self, logger=None, critical=None, error=None, + warning=None, info=None, debug=None, notset=None): + if logger is None: + logger = logging.getLogger() + elif not isinstance(logger, logging.Logger): + logger = logging.getLogger(logger) + self.logger = logger + + self.records = [] + + self.expected_count_by_level = { + logging.CRITICAL: critical, + logging.ERROR: error, + logging.WARNING: warning, + logging.INFO: info, + logging.DEBUG: debug, + logging.NOTSET: notset + } + + self._expected_count = sum([v for k, v in self.expected_count_by_level.items() if v is not None]) + """Amount of any logging expected""" + + super(LoggingValidator, self).__init__() + + def __enter__(self): + """Context (i.e., with) support""" + self.records = [] # Reset recorded LogRecords + self.logger.addHandler(self) + self.logger.propagate = False + # ensure no log message is ignored + self.entry_level = self.logger.level * 1 + self.logger.setLevel(logging.DEBUG) + self.entry_disabled = self.logger.disabled + self.logger.disabled = False + return self + + def can_be_checked(self): + """Returns True if this listener have received enough messages to + be valid, and then checked. + + This can be useful for asynchronous wait of messages. It allows process + an early break, instead of waiting much time in an active loop. + """ + return len(self.records) >= self._expected_count + + def get_count_by_level(self): + """Returns the current message count by level. + """ + count = { + logging.CRITICAL: 0, + logging.ERROR: 0, + logging.WARNING: 0, + logging.INFO: 0, + logging.DEBUG: 0, + logging.NOTSET: 0 + } + for record in self.records: + level = record.levelno + if level in count: + count[level] = count[level] + 1 + return count + + def __exit__(self, exc_type, exc_value, traceback): + """Context (i.e., with) support""" + self.logger.removeHandler(self) + self.logger.propagate = True + self.logger.setLevel(self.entry_level) + self.logger.disabled = self.entry_disabled + + count_by_level = self.get_count_by_level() + + # Remove keys which does not matter + ignored = [r for r, v in self.expected_count_by_level.items() if v is None] + expected_count_by_level = dict(self.expected_count_by_level) + for i in ignored: + del count_by_level[i] + del expected_count_by_level[i] + + if count_by_level != expected_count_by_level: + # Re-send record logs through logger as they where masked + # to help debug + message = "" + for level in count_by_level.keys(): + if message != "": + message += ", " + count = count_by_level[level] + expected_count = expected_count_by_level[level] + message += "%d %s (got %d)" % (expected_count, logging.getLevelName(level), count) + + raise LoggingRuntimeError( + 'Expected %s' % message, records=list(self.records)) + + def emit(self, record): + """Override :meth:`logging.Handler.emit`""" + self.records.append(record) + + +def validate_logging(logger=None, critical=None, error=None, + warning=None, info=None, debug=None, notset=None): + """Decorator checking number of logging messages. + + Propagation of logging messages is disabled by this decorator. + + In case the expected number of logging messages is not found, it raises + a RuntimeError. + + >>> class Test(unittest.TestCase): + ... @validate_logging('module_logger_name', error=2, warning=0) + ... def test(self): + ... pass # Test expecting 2 ERROR and 0 WARNING messages + + :param logger: Name or instance of the logger to test. + (Default: root logger) + :type logger: str or :class:`logging.Logger` + :param int critical: Expected number of CRITICAL messages. + Default: Do not check. + :param int error: Expected number of ERROR messages. + Default: Do not check. + :param int warning: Expected number of WARNING messages. + Default: Do not check. + :param int info: Expected number of INFO messages. + Default: Do not check. + :param int debug: Expected number of DEBUG messages. + Default: Do not check. + :param int notset: Expected number of NOTSET messages. + Default: Do not check. + """ + def decorator(func): + test_context = LoggingValidator( + logger, critical, error, warning, info, debug, notset) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + with test_context: + result = func(*args, **kwargs) + return result + return wrapper + return decorator + + +# Backward compatibility +class TestLogging(LoggingValidator): + def __init__(self, *args, **kwargs): + deprecation.deprecated_warning( + "Class", + "TestLogging", + since_version="1.0.0", + replacement="LoggingValidator") + super().__init__(*args, **kwargs) + + +@deprecation.deprecated(since_version="1.0.0", replacement="validate_logging") +def test_logging(*args, **kwargs): + return validate_logging(*args, **kwargs) + + +# Simulate missing library context +class EnsureImportError(object): + """This context manager allows to simulate the unavailability + of a library, even if it is actually available. It ensures that + an ImportError is raised if the code inside the context tries to + import the module. + + It can be used to test that a correct fallback library is used, + or that the expected error code is returned. + + Trivial example:: + + from silx.utils.testutils import EnsureImportError + + with EnsureImportError("h5py"): + try: + import h5py + except ImportError: + print("Good") + + .. note:: + + This context manager does not remove the library from the namespace, + if it is already imported. It only ensures that any attempt to import + it again will cause an ImportError to be raised. + """ + def __init__(self, name): + """ + + :param str name: Name of module to be hidden (e.g. "h5py") + """ + self.module_name = name + + def __enter__(self): + """Simulate failed import by setting sys.modules[name]=None""" + if self.module_name not in sys.modules: + self._delete_on_exit = True + self._backup = None + else: + self._delete_on_exit = False + self._backup = sys.modules[self.module_name] + sys.modules[self.module_name] = None + + def __exit__(self, exc_type, exc_val, exc_tb): + """Restore previous state""" + if self._delete_on_exit: + del sys.modules[self.module_name] + else: + sys.modules[self.module_name] = self._backup diff --git a/src/silx/utils/weakref.py b/src/silx/utils/weakref.py new file mode 100644 index 0000000..06646e8 --- /dev/null +++ b/src/silx/utils/weakref.py @@ -0,0 +1,361 @@ +# coding: utf-8 +# /*########################################################################## +# +# Copyright (c) 2016-2018 European Synchrotron Radiation Facility +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# ###########################################################################*/ +"""Weakref utils for compatibility between Python 2 and Python 3 or for +extended features. +""" +from __future__ import absolute_import + +__authors__ = ["V. Valls"] +__license__ = "MIT" +__date__ = "15/09/2016" + + +import weakref +import types +import inspect + + +def ref(object, callback=None): + """Returns a weak reference to object. The original object can be retrieved + by calling the reference object if the referent is still alive. If the + referent is no longer alive, calling the reference object will cause None + to be returned. + + The signature is the same as the standard `weakref` library, but it returns + `WeakMethod` if the object is a bound method. + + :param object: An object + :param func callback: If provided, and the returned weakref object is + still alive, the callback will be called when the object is about to + be finalized. The weak reference object will be passed as the only + parameter to the callback. Then the referent will no longer be + available. + :return: A weak reference to the object + """ + if inspect.ismethod(object): + return WeakMethod(object, callback) + else: + return weakref.ref(object, callback) + + +def proxy(object, callback=None): + """Return a proxy to object which uses a weak reference. This supports use + of the proxy in most contexts instead of requiring the explicit + dereferencing used with weak reference objects. + + The signature is the same as the standard `weakref` library, but it returns + `WeakMethodProxy` if the object is a bound method. + + :param object: An object + :param func callback: If provided, and the returned weakref object is + still alive, the callback will be called when the object is about to + be finalized. The weak reference object will be passed as the only + parameter to the callback. Then the referent will no longer be + available. + :return: A proxy to a weak reference of the object + """ + if inspect.ismethod(object): + return WeakMethodProxy(object, callback) + else: + return weakref.proxy(object, callback) + + +class WeakMethod(object): + """Wraps a callable object like a function or a bound method. + Feature callback when the object is about to be finalized. + Provids the same interface as a normal weak reference. + """ + + def __init__(self, function, callback=None): + """ + Constructor + :param function: Function/method to be called + :param callback: If callback is provided and not None, + and the returned weakref object is still alive, the + callback will be called when the object is about to + be finalized; the weak reference object will be passed + as the only parameter to the callback; the referent will + no longer be available + """ + self.__callback = callback + + if inspect.ismethod(function): + # it is a bound method + self.__obj = weakref.ref(function.__self__, self.__call_callback) + self.__method = weakref.ref(function.__func__, self.__call_callback) + else: + self.__obj = None + self.__method = weakref.ref(function, self.__call_callback) + + def __call_callback(self, ref): + """Called when the object is about to be finalized""" + if not self.is_alive(): + return + self.__obj = None + self.__method = None + if self.__callback is not None: + self.__callback(self) + + def __call__(self): + """Return a callable function or None if the WeakMethod is dead.""" + if self.__obj is not None: + method = self.__method() + obj = self.__obj() + if method is None or obj is None: + return None + return types.MethodType(method, obj) + elif self.__method is not None: + return self.__method() + else: + return None + + def is_alive(self): + """True if the WeakMethod is still alive""" + return self.__method is not None + + def __eq__(self, other): + """Check it another obect is equal to this. + + :param object other: Object to compare with + """ + if isinstance(other, WeakMethod): + if not self.is_alive(): + return False + return self.__obj == other.__obj and self.__method == other.__method + return False + + def __ne__(self, other): + """Check it another obect is not equal to this. + + :param object other: Object to compare with + """ + if isinstance(other, WeakMethod): + if not self.is_alive(): + return False + return self.__obj != other.__obj or self.__method != other.__method + return True + + def __hash__(self): + """Returns the hash for the object.""" + return self.__obj.__hash__() ^ self.__method.__hash__() + + +class WeakMethodProxy(WeakMethod): + """Wraps a callable object like a function or a bound method + with a weakref proxy. + """ + def __call__(self, *args, **kwargs): + """Dereference the method and call it if the method is still alive. + Else raises an ReferenceError. + + :raises: ReferenceError, if the method is not alive + """ + fn = super(WeakMethodProxy, self).__call__() + if fn is None: + raise ReferenceError("weakly-referenced object no longer exists") + return fn(*args, **kwargs) + + +class WeakList(list): + """Manage a list of weaked references. + When an object is dead, the list is flaged as invalid. + If expected the list is cleaned up to remove dead objects. + """ + + def __init__(self, enumerator=()): + """Create a WeakList + + :param iterator enumerator: A list of object to initialize the + list + """ + list.__init__(self) + self.__list = [] + self.__is_valid = True + for obj in enumerator: + self.append(obj) + + def __invalidate(self, ref): + """Flag the list as invalidated. The list contains dead references.""" + self.__is_valid = False + + def __create_ref(self, obj): + """Create a weakref from an object. It uses the `ref` module function. + """ + return ref(obj, self.__invalidate) + + def __clean(self): + """Clean the list from dead references""" + if self.__is_valid: + return + self.__list = [ref for ref in self.__list if ref() is not None] + self.__is_valid = True + + def __iter__(self): + """Iterate over objects of the list""" + for ref in self.__list: + obj = ref() + if obj is not None: + yield obj + + def __len__(self): + """Count item on the list""" + self.__clean() + return len(self.__list) + + def __getitem__(self, key): + """Returns the object at the requested index + + :param key: Indexes to get + :type key: int or slice + """ + self.__clean() + data = self.__list[key] + if isinstance(data, list): + result = [ref() for ref in data] + else: + result = data() + return result + + def __setitem__(self, key, obj): + """Set an item at an index + + :param key: Indexes to set + :type key: int or slice + """ + self.__clean() + if isinstance(key, slice): + objs = [self.__create_ref(o) for o in obj] + self.__list[key] = objs + else: + obj_ref = self.__create_ref(obj) + self.__list[key] = obj_ref + + def __delitem__(self, key): + """Delete an Indexes item of this list + + :param key: Index to delete + :type key: int or slice + """ + self.__clean() + del self.__list[key] + + def __delslice__(self, i, j): + """Looks to be used in Python 2.7""" + self.__delitem__(slice(i, j, None)) + + def __setslice__(self, i, j, sequence): + """Looks to be used in Python 2.7""" + self.__setitem__(slice(i, j, None), sequence) + + def __getslice__(self, i, j): + """Looks to be used in Python 2.7""" + return self.__getitem__(slice(i, j, None)) + + def __reversed__(self): + """Returns a copy of the reverted list""" + reversed_list = reversed(list(self)) + return WeakList(reversed_list) + + def __contains__(self, obj): + """Returns true if the object is in the list""" + ref = self.__create_ref(obj) + return ref in self.__list + + def __add__(self, other): + """Returns a WeakList containing this list an the other""" + l = WeakList(self) + l.extend(other) + return l + + def __iadd__(self, other): + """Add objects to this list inplace""" + self.extend(other) + return self + + def __mul__(self, n): + """Returns a WeakList containing n-duplication object of this list""" + return WeakList(list(self) * n) + + def __imul__(self, n): + """N-duplication of the objects to this list inplace""" + self.__list *= n + return self + + def append(self, obj): + """Add an object at the end of the list""" + ref = self.__create_ref(obj) + self.__list.append(ref) + + def count(self, obj): + """Returns the number of occurencies of an object""" + ref = self.__create_ref(obj) + return self.__list.count(ref) + + def extend(self, other): + """Append the list with all objects from another list""" + for obj in other: + self.append(obj) + + def index(self, obj): + """Returns the index of an object""" + ref = self.__create_ref(obj) + return self.__list.index(ref) + + def insert(self, index, obj): + """Insert an object at the requested index""" + ref = self.__create_ref(obj) + self.__list.insert(index, ref) + + def pop(self, index=-1): + """Remove and return an object at the requested index""" + self.__clean() + obj = self.__list.pop(index)() + return obj + + def remove(self, obj): + """Remove an object from the list""" + ref = self.__create_ref(obj) + self.__list.remove(ref) + + def reverse(self): + """Reverse the list inplace""" + self.__list.reverse() + + def sort(self, key=None, reverse=False): + """Sort the list inplace. + Not very efficient. + """ + sorted_list = list(self) + sorted_list.sort(key=key, reverse=reverse) + self.__list = [] + self.extend(sorted_list) + + def __str__(self): + unref_list = list(self) + return "WeakList(%s)" % str(unref_list) + + def __repr__(self): + unref_list = list(self) + return "WeakList(%s)" % repr(unref_list) |