diff options
Diffstat (limited to 'silx/resources/__init__.py')
| -rw-r--r-- | silx/resources/__init__.py | 231 |
1 files changed, 196 insertions, 35 deletions
diff --git a/silx/resources/__init__.py b/silx/resources/__init__.py index f5b06cb..250aed1 100644 --- a/silx/resources/__init__.py +++ b/silx/resources/__init__.py @@ -56,7 +56,7 @@ of this modules to ensure access across different distribution schemes: __authors__ = ["V.A. Sole", "Thomas Vincent", "J. Kieffer"] __license__ = "MIT" -__date__ = "02/05/2017" +__date__ = "06/09/2017" import os @@ -67,6 +67,7 @@ import getpass import logging import tempfile import unittest +import importlib from silx.third_party import six logger = logging.getLogger(__name__) @@ -100,31 +101,179 @@ if getattr(sys, 'frozen', False): _RESOURCES_DIR = _dir +class _ResourceDirectory(object): + """Store a source of resources""" + + def __init__(self, package_name, package_path=None, forced_path=None): + if forced_path is None: + if package_path is None: + if pkg_resources is None: + # In this case we have to compute the package path + # Else it will not be used + module = importlib.import_module(package_name) + package_path = os.path.abspath(os.path.dirname(module.__file__)) + self.package_name = package_name + self.package_path = package_path + self.forced_path = forced_path + + +_SILX_DIRECTORY = _ResourceDirectory( + package_name=__name__, + package_path=os.path.abspath(os.path.dirname(__file__)), + forced_path=_RESOURCES_DIR) + +_RESOURCE_DIRECTORIES = {} +_RESOURCE_DIRECTORIES["silx"] = _SILX_DIRECTORY + + +def register_resource_directory(name, package_name, forced_path=None): + """Register another resource directory to the available list. + + By default only the directory "silx" is available. + + .. versionadded:: 0.6 + + :param str name: Name of the resource directory. It is used on the resource + name to specify the resource directory to use. The resource + "silx:foo.png" will use the "silx" resource directory. + :param str package_name: Python name of the package containing resources. + For example "silx.resources". + :param str forced_path: Path containing the resources. If specified + `pkg_resources` nor `package_name` will be used + For example "silx.resources". + :raises ValueError: If the resource directory name already exists. + """ + if name in _RESOURCE_DIRECTORIES: + raise ValueError("Resource directory name %s already exists" % name) + resource_directory = _ResourceDirectory( + package_name=package_name, + forced_path=forced_path) + _RESOURCE_DIRECTORIES[name] = resource_directory + + +def list_dir(resource): + """List the content of a resource directory. + + Result are not prefixed by the resource name. + + The resource name can be prefixed by the name of a resource directory. For + example "silx:foo.png" identify the resource "foo.png" from the resource + directory "silx". See also :func:`register_resource_directory`. + + :param str resource: Name of the resource directory to list + :return: list of name contained in the directory + :rtype: list + """ + resource_directory, resource_name = _get_package_and_resource(resource) + + if resource_directory.forced_path is not None: + # if set, use this directory + path = resource_filename(resource) + return os.listdir(path) + elif pkg_resources is None: + # Fallback if pkg_resources is not available + path = resource_filename(resource) + return os.listdir(path) + else: + # Preferred way to get resources as it supports zipfile package + package_name = resource_directory.package_name + return pkg_resources.resource_listdir(package_name, resource_name) + + +def is_dir(resource): + """True is the resource is a resource directory. + + The resource name can be prefixed by the name of a resource directory. For + example "silx:foo.png" identify the resource "foo.png" from the resource + directory "silx". See also :func:`register_resource_directory`. + + :param str resource: Name of the resource + :rtype: bool + """ + path = resource_filename(resource) + return os.path.isdir(path) + + +def _get_package_and_resource(resource, default_directory=None): + """ + Return the resource directory class and a cleaned resource name without + prefix. + + :param str: resource: Name of the resource with resource prefix. + :param str default_directory: If the resource is not prefixed, the resource + will be searched on this default directory of the silx resource + directory. + :rtype: tuple(_ResourceDirectory, str) + :raises ValueError: If the resource name uses an unregistred resource + directory name + """ + if ":" in resource: + prefix, resource = resource.split(":", 1) + else: + prefix = "silx" + if default_directory is not None: + resource = os.path.join(default_directory, resource) + if prefix not in _RESOURCE_DIRECTORIES: + raise ValueError("Resource '%s' uses an unregistred prefix", resource) + resource_directory = _RESOURCE_DIRECTORIES[prefix] + return resource_directory, resource + + def resource_filename(resource): """Return filename corresponding to resource. - resource can be the name of either a file or a directory. The existence of the resource is not checked. + The resource name can be prefixed by the name of a resource directory. For + example "silx:foo.png" identify the resource "foo.png" from the resource + directory "silx". See also :func:`register_resource_directory`. + + :param str resource: Resource path relative to resource directory + using '/' path separator. It can be either a file or + a directory. + :raises ValueError: If the resource name uses an unregistred resource + directory name + :return: Absolute resource path in the file system + :rtype: str + """ + return _resource_filename(resource, default_directory=None) + + +def _resource_filename(resource, default_directory=None): + """Return filename corresponding to resource. + + The existence of the resource is not checked. + + The resource name can be prefixed by the name of a resource directory. For + example "silx:foo.png" identify the resource "foo.png" from the resource + directory "silx". See also :func:`register_resource_directory`. + :param str resource: Resource path relative to resource directory - using '/' path separator. + using '/' path separator. It can be either a file or + a directory. + :param str default_directory: If the resource is not prefixed, the resource + will be searched on this default directory of the silx resource + directory. It should only be used internally by silx. :return: Absolute resource path in the file system + :rtype: str """ - # Not in use, uncomment when functionality is needed - # If _RESOURCES_DOC_DIR is set, use it to get resources in doc/ subfoldler - # from an alternative directory. - # if _RESOURCES_DOC_DIR is not None and (resource is 'doc' or - # resource.startswith('doc/')): - # # Remove doc folder from resource relative path - # return os.path.join(_RESOURCES_DOC_DIR, *resource.split('/')[1:]) - - if _RESOURCES_DIR is not None: # if set, use this directory - return os.path.join(_RESOURCES_DIR, *resource.split('/')) - elif pkg_resources is None: # Fallback if pkg_resources is not available - return os.path.join(os.path.abspath(os.path.dirname(__file__)), - *resource.split('/')) - else: # Preferred way to get resources as it supports zipfile package - return pkg_resources.resource_filename(__name__, resource) + resource_directory, resource_name = _get_package_and_resource(resource, + default_directory=default_directory) + + if resource_directory.forced_path is not None: + # if set, use this directory + base_dir = resource_directory.forced_path + resource_path = os.path.join(base_dir, *resource_name.split('/')) + return resource_path + elif pkg_resources is None: + # Fallback if pkg_resources is not available + base_dir = resource_directory.package_path + resource_path = os.path.join(base_dir, *resource_name.split('/')) + return resource_path + else: + # Preferred way to get resources as it supports zipfile package + package_name = resource_directory.package_name + return pkg_resources.resource_filename(package_name, resource_name) class ExternalResources(object): @@ -139,20 +288,26 @@ class ExternalResources(object): timeout=60): """Constructor of the class - :param project: name of the project, like "silx" - :param url_base: base URL for the data, like "http://www.silx.org/pub" - :param env_key: name of the environment variable which contains the - test_data directory like "SILX_DATA" + :param str project: name of the project, like "silx" + :param str url_base: base URL for the data, like "http://www.silx.org/pub" + :param str env_key: name of the environment variable which contains the + test_data directory, like "SILX_DATA". + If None (default), then the name of the + environment variable is built from the project argument: + "<PROJECT>_DATA". + The environment variable is optional: in case it is not set, + a directory in the temporary folder is used. :param timeout: time in seconds before it breaks """ self.project = project self._initialized = False self._tempdir = None self.sem = threading.Semaphore() - self.env_key = env_key + self.env_key = env_key or (self.project.upper() + "_DATA") self.url_base = url_base self.all_data = set() self.timeout = timeout + self.data_home = None def _initialize_tmpdir(self): """Initialize the temporary directory""" @@ -202,7 +357,7 @@ class ExternalResources(object): self._tempdir = None def getfile(self, filename): - """Downloads the requested file from web-server available + """Downloads the requested file from web-server available at https://www.silx.org/pub/silx/ :param: relative name of the image. @@ -249,12 +404,13 @@ class ExternalResources(object): data to disk at %s" % self.data_home) if not os.path.isfile(fullfilename): - raise RuntimeError("Could not automatically \ - download test images %s!\n \ If you are behind a firewall, \ - please set both environment variable http_proxy and https_proxy.\ - This even works under windows ! \n \ - Otherwise please try to download the images manually from \n%s/%s"\ - % (filename, self.url_base, filename)) + raise RuntimeError( + "Could not automatically \ + download test images %s!\n \ If you are behind a firewall, \ + please set both environment variable http_proxy and https_proxy.\ + This even works under windows ! \n \ + Otherwise please try to download the images manually from \n%s/%s" + % (filename, self.url_base, filename)) if filename not in self.all_data: self.all_data.add(filename) @@ -269,19 +425,19 @@ class ExternalResources(object): return fullfilename def getdir(self, dirname): - """Downloads the requested tarball from the server + """Downloads the requested tarball from the server https://www.silx.org/pub/silx/ and unzips it into the data directory :param: relative name of the image. - :return: full path of the locally saved file. + :return: list of files with their full path. """ lodn = dirname.lower() if (lodn.endswith("tar") or lodn.endswith("tgz") or lodn.endswith("tbz2") or lodn.endswith("tar.gz") or lodn.endswith("tar.bz2")): import tarfile - engine = tarfile.TarFile + engine = tarfile.TarFile.open elif lodn.endswith("zip"): import zipfile engine = zipfile.ZipFile @@ -289,9 +445,14 @@ class ExternalResources(object): raise RuntimeError("Unsupported archive format. Only tar and zip " "are currently supported") full_path = self.getfile(dirname) - with engine.open(full_path) as fd: + root = os.path.dirname(full_path) + with engine(full_path, mode="r") as fd: fd.extractall(self.data_home) - return full_path + if lodn.endswith("zip"): + result = [os.path.join(root, i) for i in fd.namelist()] + else: + result = [os.path.join(root, i) for i in fd.getnames()] + return result def download_all(self, imgs=None): """Download all data needed for the test/benchmarks |