New upstream version 0.6.0

17 files changed, 304 insertions, 207 deletions

diff --git a/.github/workflows/ci_workflows.yml b/.github/workflows/ci_workflows.yml
index 286cdb6..179eb53 100644
--- a/.github/workflows/ci_workflows.yml
+++ b/.github/workflows/ci_workflows.yml
@@ -7,47 +7,34 @@ on:
     # Run every Sunday at 06:53 UTC
     - cron: 53 6 * * 0
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   tests:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            python-version: 3.7
-            toxenv: py37-test-pytest46
-          - os: windows-latest
-            python-version: 3.7
-            toxenv: py37-test-pytest50
-          - os: macos-latest
-            python-version: 3.8
-            toxenv: py38-test-pytest52
-          - os: ubuntu-latest
-            python-version: 3.8
-            toxenv: py38-test-pytest53
-          - os: windows-latest
-            python-version: 3.9
-            toxenv: py39-test-pytest60
-          - os: macos-latest
-            python-version: 3.9
-            toxenv: py39-test-pytest61
-          - os: ubuntu-latest
-            python-version: '3.10'
-            toxenv: py310-test-pytest62
-          - os: ubuntu-latest
-            python-version: '3.10'
-            toxenv: py310-test-pytestdev
-
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install tox
-      run: python -m pip install tox
-    - name: Run tox
-      run: tox -v -e ${{ matrix.toxenv }}
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1
+    with:
+      envs: |
+        - linux: codestyle
+        - linux: py37-test-pytestoldest
+        - macos: py37-test-pytest50
+        - windows: py38-test-pytest52
+        - linux: py38-test-pytest53
+        - macos: py39-test-pytest60
+        - windows: py39-test-pytest61
+        - linux: py310-test-pytest62
+        - macos: py310-test-pytest70
+        - windows: py310-test-pytest71
+        - linux: py311-test-pytest72
+        - macos: py311-test-pytest73
+        - windows: py312-test-pytest74
+        - linux: py312-test-devdeps
+  publish:
+    needs: tests
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1
+    with:
+      test_extras: test
+      test_command: pytest $GITHUB_WORKSPACE/tests; pytest --arraydiff $GITHUB_WORKSPACE/tests
+    secrets:
+      pypi_token: ${{ secrets.pypi_password }}
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
deleted file mode 100644
index 2631bf7..0000000
--- a/.github/workflows/publish.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-name: Release
-
-on:
-  pull_request:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  build-n-publish:
-    name: Build and publish Python 🐍 distributions 📦 to PyPI
-    runs-on: ubuntu-latest
-    if: ((github.event_name == 'push' && startsWith(github.ref, 'refs/tags')) || contains(github.event.pull_request.labels.*.name, 'Build wheels'))
-
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-
-    - name: Install python-build and twine
-      run: python -m pip install pip build "twine>=3.3" -U
-
-    - name: Build package
-      run: python -m build --sdist --wheel .
-
-    - name: List result
-      run: ls -l dist
-
-    - name: Check long_description
-      run: python -m twine check --strict dist/*
-
-    # FIXME: pytest not found
-    #- name: Test package
-    #  run: |
-    #    cd ..
-    #    python -m venv testenv
-    #    testenv/bin/pip install pytest pytest-arraydiff/dist/*.whl
-    #    testenv/bin/pytest pytest-arraydiff/tests --arraydiff
-
-    - name: Publish distribution 📦 to PyPI
-      if: startsWith(github.ref, 'refs/tags')
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
diff --git a/CHANGES.md b/CHANGES.md
index 3e86ce3..f78d710 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,13 @@
+0.6 (2023-11-15)
+----------------
+
+- Add ability to compare to Pandas DataFrames and store them as HDF5 files [#23]
+
+- Fix ``array_compare`` so that the ``atol`` parameter is correctly used with
+  FITS files. [#33]
+
+- Test inside ``pytest_runtest_call`` hook. [#36]
+
 0.5 (2022-01-12)
 ----------------
 
diff --git a/PKG-INFO b/PKG-INFO
index 7d951cd..096ef1a 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: pytest-arraydiff
-Version: 0.5.0
+Version: 0.6.0
 Summary: pytest plugin to help with comparing array output from tests
 Home-page: https://github.com/astropy/pytest-arraydiff
 Author: The Astropy Developers
 Author-email: astropy.team@gmail.com
 License: BSD
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Framework :: Pytest
 Classifier: Intended Audience :: Developers
@@ -15,16 +14,24 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Utilities
 Requires-Python: >=3.7
 Description-Content-Type: text/x-rst
-Provides-Extra: test
 License-File: LICENSE
+Requires-Dist: pytest>=4.6
+Requires-Dist: numpy
+Provides-Extra: test
+Requires-Dist: astropy; extra == "test"
+Requires-Dist: pandas; extra == "test"
+Requires-Dist: tables; extra == "test"
 
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5811772.svg
    :target: https://doi.org/10.5281/zenodo.5811772
@@ -43,7 +50,8 @@ in cases where the arrays are too large to conveniently hard-code them
 in the tests (e.g. ``np.testing.assert_allclose(x, [1, 2, 3])``).
 
 The basic idea is that you can write a test that generates a Numpy array (or
-other related objects depending on the format). You can then either run the
+other related objects depending on the format, e.g. pandas DataFrame).
+You can then either run the
 tests in a mode to **generate** reference files from the arrays, or you can run
 the tests in **comparison** mode, which will compare the results of the tests to
 the reference ones within some tolerance.
@@ -53,6 +61,7 @@ At the moment, the supported file formats for the reference files are:
 -  A plain text-based format (based on Numpy ``loadtxt`` output)
 -  The FITS format (requires `astropy <http://www.astropy.org>`__). With this
    format, tests can return either a Numpy array for a FITS HDU object.
+-  A pandas HDF5 format using the pandas HDFStore
 
 For more information on how to write tests to do this, see the **Using**
 section below.
@@ -137,12 +146,12 @@ The supported formats at this time are ``text`` and ``fits``, and
 contributions for other formats are welcome. The default format is
 ``text``.
 
-Another argument is the relative tolerance for floating point values
-(which defaults to 1e-7):
+Additional arguments are the relative and absolute tolerances for floating
+point values (which default to 1e-7 and 0, respectively):
 
 .. code:: python
 
-    @pytest.mark.array_compare(rtol=20)
+    @pytest.mark.array_compare(rtol=20, atol=0.1)
     def test_array():
         ...
 
@@ -220,5 +229,3 @@ install the latest version of the plugin then do::
 
 The reason for having to install the plugin first is to ensure that the
 plugin is correctly loaded as part of the test suite.
-
-
diff --git a/README.rst b/README.rst
index 3ac149a..48136ea 100644
--- a/README.rst
+++ b/README.rst
@@ -15,7 +15,8 @@ in cases where the arrays are too large to conveniently hard-code them
 in the tests (e.g. ``np.testing.assert_allclose(x, [1, 2, 3])``).
 
 The basic idea is that you can write a test that generates a Numpy array (or
-other related objects depending on the format). You can then either run the
+other related objects depending on the format, e.g. pandas DataFrame).
+You can then either run the
 tests in a mode to **generate** reference files from the arrays, or you can run
 the tests in **comparison** mode, which will compare the results of the tests to
 the reference ones within some tolerance.
@@ -25,6 +26,7 @@ At the moment, the supported file formats for the reference files are:
 -  A plain text-based format (based on Numpy ``loadtxt`` output)
 -  The FITS format (requires `astropy <http://www.astropy.org>`__). With this
    format, tests can return either a Numpy array for a FITS HDU object.
+-  A pandas HDF5 format using the pandas HDFStore
 
 For more information on how to write tests to do this, see the **Using**
 section below.
@@ -109,12 +111,12 @@ The supported formats at this time are ``text`` and ``fits``, and
 contributions for other formats are welcome. The default format is
 ``text``.
 
-Another argument is the relative tolerance for floating point values
-(which defaults to 1e-7):
+Additional arguments are the relative and absolute tolerances for floating
+point values (which default to 1e-7 and 0, respectively):
 
 .. code:: python
 
-    @pytest.mark.array_compare(rtol=20)
+    @pytest.mark.array_compare(rtol=20, atol=0.1)
     def test_array():
         ...
 
diff --git a/pytest_arraydiff.egg-info/PKG-INFO b/pytest_arraydiff.egg-info/PKG-INFO
index 7d951cd..096ef1a 100644
--- a/pytest_arraydiff.egg-info/PKG-INFO
+++ b/pytest_arraydiff.egg-info/PKG-INFO
@@ -1,12 +1,11 @@
 Metadata-Version: 2.1
 Name: pytest-arraydiff
-Version: 0.5.0
+Version: 0.6.0
 Summary: pytest plugin to help with comparing array output from tests
 Home-page: https://github.com/astropy/pytest-arraydiff
 Author: The Astropy Developers
 Author-email: astropy.team@gmail.com
 License: BSD
-Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Framework :: Pytest
 Classifier: Intended Audience :: Developers
@@ -15,16 +14,24 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.6
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Topic :: Software Development :: Testing
 Classifier: Topic :: Utilities
 Requires-Python: >=3.7
 Description-Content-Type: text/x-rst
-Provides-Extra: test
 License-File: LICENSE
+Requires-Dist: pytest>=4.6
+Requires-Dist: numpy
+Provides-Extra: test
+Requires-Dist: astropy; extra == "test"
+Requires-Dist: pandas; extra == "test"
+Requires-Dist: tables; extra == "test"
 
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5811772.svg
    :target: https://doi.org/10.5281/zenodo.5811772
@@ -43,7 +50,8 @@ in cases where the arrays are too large to conveniently hard-code them
 in the tests (e.g. ``np.testing.assert_allclose(x, [1, 2, 3])``).
 
 The basic idea is that you can write a test that generates a Numpy array (or
-other related objects depending on the format). You can then either run the
+other related objects depending on the format, e.g. pandas DataFrame).
+You can then either run the
 tests in a mode to **generate** reference files from the arrays, or you can run
 the tests in **comparison** mode, which will compare the results of the tests to
 the reference ones within some tolerance.
@@ -53,6 +61,7 @@ At the moment, the supported file formats for the reference files are:
 -  A plain text-based format (based on Numpy ``loadtxt`` output)
 -  The FITS format (requires `astropy <http://www.astropy.org>`__). With this
    format, tests can return either a Numpy array for a FITS HDU object.
+-  A pandas HDF5 format using the pandas HDFStore
 
 For more information on how to write tests to do this, see the **Using**
 section below.
@@ -137,12 +146,12 @@ The supported formats at this time are ``text`` and ``fits``, and
 contributions for other formats are welcome. The default format is
 ``text``.
 
-Another argument is the relative tolerance for floating point values
-(which defaults to 1e-7):
+Additional arguments are the relative and absolute tolerances for floating
+point values (which default to 1e-7 and 0, respectively):
 
 .. code:: python
 
-    @pytest.mark.array_compare(rtol=20)
+    @pytest.mark.array_compare(rtol=20, atol=0.1)
     def test_array():
         ...
 
@@ -220,5 +229,3 @@ install the latest version of the plugin then do::
 
 The reason for having to install the plugin first is to ensure that the
 plugin is correctly loaded as part of the test suite.
-
-
diff --git a/pytest_arraydiff.egg-info/SOURCES.txt b/pytest_arraydiff.egg-info/SOURCES.txt
index 20b25e4..0a7bebd 100644
--- a/pytest_arraydiff.egg-info/SOURCES.txt
+++ b/pytest_arraydiff.egg-info/SOURCES.txt
@@ -8,7 +8,6 @@ setup.cfg
 setup.py
 tox.ini
 .github/workflows/ci_workflows.yml
-.github/workflows/publish.yml
 pytest_arraydiff/__init__.py
 pytest_arraydiff/plugin.py
 pytest_arraydiff/version.py
@@ -20,9 +19,11 @@ pytest_arraydiff.egg-info/not-zip-safe
 pytest_arraydiff.egg-info/requires.txt
 pytest_arraydiff.egg-info/top_level.txt
 tests/test_pytest_arraydiff.py
+tests/baseline/test_absolute_tolerance.fits
+tests/baseline/test_relative_tolerance.fits
 tests/baseline/test_succeeds_class.fits
 tests/baseline/test_succeeds_func_default.txt
 tests/baseline/test_succeeds_func_fits.fits
 tests/baseline/test_succeeds_func_fits_hdu.fits
-tests/baseline/test_succeeds_func_text.txt
-tests/baseline/test_tolerance.fits
\ No newline at end of file
+tests/baseline/test_succeeds_func_pdhdf.h5
+tests/baseline/test_succeeds_func_text.txt
\ No newline at end of file
diff --git a/pytest_arraydiff.egg-info/entry_points.txt b/pytest_arraydiff.egg-info/entry_points.txt
index 2dc9ed1..0dfa4f6 100644
--- a/pytest_arraydiff.egg-info/entry_points.txt
+++ b/pytest_arraydiff.egg-info/entry_points.txt
@@ -1,3 +1,2 @@
 [pytest11]
 pytest_arraydiff = pytest_arraydiff.plugin
-
diff --git a/pytest_arraydiff.egg-info/requires.txt b/pytest_arraydiff.egg-info/requires.txt
index 9ce3e63..f35d289 100644
--- a/pytest_arraydiff.egg-info/requires.txt
+++ b/pytest_arraydiff.egg-info/requires.txt
@@ -3,3 +3,5 @@ numpy
 
 [test]
 astropy
+pandas
+tables
diff --git a/pytest_arraydiff/plugin.py b/pytest_arraydiff/plugin.py
index 8d54faf..f476e68 100755
--- a/pytest_arraydiff/plugin.py
+++ b/pytest_arraydiff/plugin.py
@@ -28,9 +28,6 @@
 #
 #   https://github.com/astrofrog/pytest-mpl
 
-
-from functools import wraps
-
 import os
 import abc
 import shutil
@@ -116,8 +113,9 @@ class FITSDiff(BaseDiff):
         from astropy.io.fits.diff import FITSDiff
         from astropy.utils.introspection import minversion
         if minversion(astropy, '2.0'):
-            diff = FITSDiff(reference_file, test_file, rtol=rtol)
+            diff = FITSDiff(reference_file, test_file, rtol=rtol, atol=atol)
         else:
+            # `atol` is not supported prior to Astropy 2.0
             diff = FITSDiff(reference_file, test_file, tolerance=rtol)
         return diff.identical, diff.report()
 
@@ -137,9 +135,43 @@ class TextDiff(SimpleArrayDiff):
         return np.savetxt(filename, data, **kwargs)
 
 
+class PDHDFDiff(BaseDiff):
+
+    extension = 'h5'
+
+    @staticmethod
+    def read(filename):
+        import pandas as pd
+        return pd.read_hdf(filename)
+
+    @staticmethod
+    def write(filename, data, **kwargs):
+        import pandas as pd  # noqa: F401
+        key = os.path.basename(filename).replace('.h5', '')
+        return data.to_hdf(filename, key=key, **kwargs)
+
+    @classmethod
+    def compare(cls, reference_file, test_file, atol=None, rtol=None):
+        import pandas.testing as pdt
+        import pandas as pd
+
+        ref_data = pd.read_hdf(reference_file)
+        test_data = pd.read_hdf(test_file)
+        try:
+            pdt.assert_frame_equal(ref_data, test_data)
+        except AssertionError as exc:
+            message = "\n\na: {0}".format(test_file) + '\n'
+            message += "b: {0}".format(reference_file) + '\n'
+            message += exc.args[0]
+            return False, message
+        else:
+            return True, ""
+
+
 FORMATS = {}
 FORMATS['fits'] = FITSDiff
 FORMATS['text'] = TextDiff
+FORMATS['pd_hdf'] = PDHDFDiff
 
 
 def _download_file(url):
@@ -166,7 +198,7 @@ def pytest_addoption(parser):
 def pytest_configure(config):
     config.getini('markers').append(
         'array_compare: for functions using array comparison')
-    
+
     if config.getoption("--arraydiff") or config.getoption("--arraydiff-generate-path") is not None:
 
         reference_dir = config.getoption("--arraydiff-reference-path")
@@ -186,6 +218,37 @@ def pytest_configure(config):
                                                       reference_dir=reference_dir,
                                                       generate_dir=generate_dir,
                                                       default_format=default_format))
+    else:
+        config.pluginmanager.register(ArrayInterceptor(config))
+
+
+def generate_test_name(item):
+    """
+    Generate a unique name for this test.
+    """
+    if item.cls is not None:
+        name = f"{item.module.__name__}.{item.cls.__name__}.{item.name}"
+    else:
+        name = f"{item.module.__name__}.{item.name}"
+    return name
+
+
+def wrap_array_interceptor(plugin, item):
+    """
+    Intercept and store arrays returned by test functions.
+    """
+    # Only intercept array on marked array tests
+    if item.get_closest_marker('array_compare') is not None:
+
+        # Use the full test name as a key to ensure correct array is being retrieved
+        test_name = generate_test_name(item)
+
+        def array_interceptor(store, obj):
+            def wrapper(*args, **kwargs):
+                store.return_value[test_name] = obj(*args, **kwargs)
+            return wrapper
+
+        item.obj = array_interceptor(plugin, item.obj)
 
 
 class ArrayComparison(object):
@@ -195,12 +258,15 @@ class ArrayComparison(object):
         self.reference_dir = reference_dir
         self.generate_dir = generate_dir
         self.default_format = default_format
+        self.return_value = {}
 
-    def pytest_runtest_setup(self, item):
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_call(self, item):
 
         compare = item.get_closest_marker('array_compare')
 
         if compare is None:
+            yield
             return
 
         file_format = compare.kwargs.get('file_format', self.default_format)
@@ -220,85 +286,95 @@ class ArrayComparison(object):
 
         write_kwargs = compare.kwargs.get('write_kwargs', {})
 
-        original = item.function
+        reference_dir = compare.kwargs.get('reference_dir', None)
+        if reference_dir is None:
+            if self.reference_dir is None:
+                reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), 'reference')
+            else:
+                reference_dir = self.reference_dir
+        else:
+            if not reference_dir.startswith(('http://', 'https://')):
+                reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), reference_dir)
 
-        @wraps(item.function)
-        def item_function_wrapper(*args, **kwargs):
+        baseline_remote = reference_dir.startswith('http')
 
-            reference_dir = compare.kwargs.get('reference_dir', None)
-            if reference_dir is None:
-                if self.reference_dir is None:
-                    reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), 'reference')
-                else:
-                    reference_dir = self.reference_dir
-            else:
-                if not reference_dir.startswith(('http://', 'https://')):
-                    reference_dir = os.path.join(os.path.dirname(item.fspath.strpath), reference_dir)
-
-            baseline_remote = reference_dir.startswith('http')
-
-            # Run test and get figure object
-            import inspect
-            if inspect.ismethod(original):  # method
-                array = original(*args[1:], **kwargs)
-            else:  # function
-                array = original(*args, **kwargs)
-
-            # Find test name to use as plot name
-            filename = compare.kwargs.get('filename', None)
-            if filename is None:
-                if single_reference:
-                    filename = original.__name__ + '.' + extension
-                else:
-                    filename = item.name + '.' + extension
-                    filename = filename.replace('[', '_').replace(']', '_')
-                    filename = filename.replace('_.' + extension, '.' + extension)
-
-            # What we do now depends on whether we are generating the reference
-            # files or simply running the test.
-            if self.generate_dir is None:
-
-                # Save the figure
-                result_dir = tempfile.mkdtemp()
-                test_array = os.path.abspath(os.path.join(result_dir, filename))
-
-                FORMATS[file_format].write(test_array, array, **write_kwargs)
-
-                # Find path to baseline array
-                if baseline_remote:
-                    baseline_file_ref = _download_file(reference_dir + filename)
-                else:
-                    baseline_file_ref = os.path.abspath(os.path.join(os.path.dirname(item.fspath.strpath), reference_dir, filename))
-
-                if not os.path.exists(baseline_file_ref):
-                    raise Exception("""File not found for comparison test
-                                    Generated file:
-                                    \t{test}
-                                    This is expected for new tests.""".format(
-                        test=test_array))
-
-                # setuptools may put the baseline arrays in non-accessible places,
-                # copy to our tmpdir to be sure to keep them in case of failure
-                baseline_file = os.path.abspath(os.path.join(result_dir, 'reference-' + filename))
-                shutil.copyfile(baseline_file_ref, baseline_file)
-
-                identical, msg = FORMATS[file_format].compare(baseline_file, test_array, atol=atol, rtol=rtol)
-
-                if identical:
-                    shutil.rmtree(result_dir)
-                else:
-                    raise Exception(msg)
+        # Run test and get array object
+        wrap_array_interceptor(self, item)
+        yield
+        test_name = generate_test_name(item)
+        if test_name not in self.return_value:
+            # Test function did not complete successfully
+            return
+        array = self.return_value[test_name]
+
+        # Find test name to use as plot name
+        filename = compare.kwargs.get('filename', None)
+        if filename is None:
+            filename = item.name + '.' + extension
+            if not single_reference:
+                filename = filename.replace('[', '_').replace(']', '_')
+                filename = filename.replace('_.' + extension, '.' + extension)
+
+        # What we do now depends on whether we are generating the reference
+        # files or simply running the test.
+        if self.generate_dir is None:
+
+            # Save the figure
+            result_dir = tempfile.mkdtemp()
+            test_array = os.path.abspath(os.path.join(result_dir, filename))
+
+            FORMATS[file_format].write(test_array, array, **write_kwargs)
 
+            # Find path to baseline array
+            if baseline_remote:
+                baseline_file_ref = _download_file(reference_dir + filename)
             else:
+                baseline_file_ref = os.path.abspath(os.path.join(os.path.dirname(item.fspath.strpath), reference_dir, filename))
 
-                if not os.path.exists(self.generate_dir):
-                    os.makedirs(self.generate_dir)
+            if not os.path.exists(baseline_file_ref):
+                raise Exception("""File not found for comparison test
+                                Generated file:
+                                \t{test}
+                                This is expected for new tests.""".format(
+                    test=test_array))
 
-                FORMATS[file_format].write(os.path.abspath(os.path.join(self.generate_dir, filename)), array, **write_kwargs)
+            # setuptools may put the baseline arrays in non-accessible places,
+            # copy to our tmpdir to be sure to keep them in case of failure
+            baseline_file = os.path.abspath(os.path.join(result_dir, 'reference-' + filename))
+            shutil.copyfile(baseline_file_ref, baseline_file)
 
-                pytest.skip("Skipping test, since generating data")
+            identical, msg = FORMATS[file_format].compare(baseline_file, test_array, atol=atol, rtol=rtol)
+
+            if identical:
+                shutil.rmtree(result_dir)
+            else:
+                raise Exception(msg)
 
-        if item.cls is not None:
-            setattr(item.cls, item.function.__name__, item_function_wrapper)
         else:
-            item.obj = item_function_wrapper
+
+            if not os.path.exists(self.generate_dir):
+                os.makedirs(self.generate_dir)
+
+            FORMATS[file_format].write(os.path.abspath(os.path.join(self.generate_dir, filename)), array, **write_kwargs)
+
+            pytest.skip("Skipping test, since generating data")
+
+
+class ArrayInterceptor:
+    """
+    This is used in place of ArrayComparison when the array comparison option is not used,
+    to make sure that we still intercept arrays returned by tests.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.return_value = {}
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_runtest_call(self, item):
+
+        if item.get_closest_marker('array_compare') is not None:
+            wrap_array_interceptor(self, item)
+
+        yield
+        return
diff --git a/pytest_arraydiff/version.py b/pytest_arraydiff/version.py
index 7856683..529d24f 100644
--- a/pytest_arraydiff/version.py
+++ b/pytest_arraydiff/version.py
@@ -1,5 +1,16 @@
-# coding: utf-8
 # file generated by setuptools_scm
 # don't change, don't track in version control
-version = '0.5.0'
-version_tuple = (0, 5, 0)
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.6.0'
+__version_tuple__ = version_tuple = (0, 6, 0)
diff --git a/setup.cfg b/setup.cfg
index ef3a9e2..5be4577 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -12,9 +12,12 @@ classifiers =
 	Programming Language :: Python
 	Programming Language :: Python :: 3
 	Programming Language :: Python :: 3 :: Only
-	Programming Language :: Python :: 3.6
 	Programming Language :: Python :: 3.7
 	Programming Language :: Python :: 3.8
+	Programming Language :: Python :: 3.9
+	Programming Language :: Python :: 3.10
+	Programming Language :: Python :: 3.11
+	Programming Language :: Python :: 3.12
 	Programming Language :: Python :: Implementation :: CPython
 	Topic :: Software Development :: Testing
 	Topic :: Utilities
@@ -36,6 +39,8 @@ install_requires =
 [options.extras_require]
 test = 
 	astropy
+	pandas
+	tables
 
 [options.entry_points]
 pytest11 = 
@@ -47,6 +52,9 @@ testpaths = tests
 xfail_strict = true
 markers = 
 	array_compare: for functions using array comparison
+filterwarnings = 
+	error
+	ignore:distutils Version classes are deprecated
 
 [flake8]
 max-line-length = 150
diff --git a/tests/baseline/test_tolerance.fits b/tests/baseline/test_absolute_tolerance.fits
index 9fab1ae..9fab1ae 100644
--- a/tests/baseline/test_tolerance.fits
+++ b/tests/baseline/test_absolute_tolerance.fits
diff --git a/tests/baseline/test_relative_tolerance.fits b/tests/baseline/test_relative_tolerance.fits
new file mode 100644
index 0000000..9fab1ae
--- /dev/null
+++ b/tests/baseline/test_relative_tolerance.fits
diff --git a/tests/baseline/test_succeeds_func_pdhdf.h5 b/tests/baseline/test_succeeds_func_pdhdf.h5
new file mode 100644
index 0000000..7e17ccd
--- /dev/null
+++ b/tests/baseline/test_succeeds_func_pdhdf.h5
diff --git a/tests/test_pytest_arraydiff.py b/tests/test_pytest_arraydiff.py
index 9749b27..5ad20dc 100644
--- a/tests/test_pytest_arraydiff.py
+++ b/tests/test_pytest_arraydiff.py
@@ -4,6 +4,9 @@ import tempfile
 
 import pytest
 import numpy as np
+from packaging.version import Version
+
+NUMPY_LT_2_0 = Version(np.__version__) < Version("2.0.dev")
 
 reference_dir = 'baseline'
 
@@ -18,6 +21,14 @@ def test_succeeds_func_text():
     return np.arange(3 * 5).reshape((3, 5))
 
 
+@pytest.mark.skipif(not NUMPY_LT_2_0, reason="AttributeError: `np.unicode_` was removed in the NumPy 2.0 release. Use `np.str_` instead.")
+@pytest.mark.array_compare(file_format='pd_hdf', reference_dir=reference_dir)
+def test_succeeds_func_pdhdf():
+    pd = pytest.importorskip('pandas')
+    return pd.DataFrame(data=np.arange(20, dtype='int64'),
+                        columns=['test_data'])
+
+
 @pytest.mark.array_compare(file_format='fits', reference_dir=reference_dir)
 def test_succeeds_func_fits():
     return np.arange(3 * 5).reshape((3, 5)).astype(np.int64)
@@ -125,9 +136,20 @@ def test_default_format(file_format):
     assert os.path.exists(os.path.join(gen_dir, 'test_default.' + ('fits' if file_format == 'fits' else 'txt')))
 
 
-@pytest.mark.array_compare(reference_dir=reference_dir, rtol=0.5, file_format='fits')
-def test_tolerance():
-    return np.ones((3, 4)) * 1.6
+@pytest.mark.array_compare(reference_dir=reference_dir, rtol=0.5,
+        file_format='fits')
+def test_relative_tolerance():
+    # Scale up the output values by 1.5 to ensure the large `rtol` value is
+    # needed. (The comparison file contains all 1.6.)
+    return np.ones((3, 4)) * 1.6 * 1.5
+
+
+@pytest.mark.array_compare(reference_dir=reference_dir, atol=1.5,
+        file_format='fits')
+def test_absolute_tolerance():
+    # Increase the output values by 1.4 to ensure the large `atol` value is
+    # needed. (The comparison file contains all 1.6.)
+    return np.ones((3, 4)) * 1.6 + 1.4
 
 
 def test_nofile():
diff --git a/tox.ini b/tox.ini
index c278ab9..bde33c9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,6 @@
 [tox]
 envlist =
-    py{37,38,39,310}-test{,-devdeps}
+    py{37,38,39,310,311,312}-test{,-pytestoldest,-pytest50,-pytest52,-pytest53,-pytest60,-pytest61,-pytest62,-pytest70,-pytest71,-pytest72,-pytest73,-pytest74,-devdeps}
     codestyle
 requires =
     setuptools >= 30.3.0
@@ -9,25 +9,38 @@ isolated_build = true
 
 [testenv]
 changedir = .tmp/{envname}
+setenv =
+    devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
 description = run tests
 deps =
-    pytest46: pytest==4.6.*
+    pytestoldest: pytest==4.6.0
     pytest50: pytest==5.0.*
     pytest52: pytest==5.2.*
     pytest53: pytest==5.3.*
     pytest60: pytest==6.0.*
     pytest61: pytest==6.1.*
     pytest62: pytest==6.2.*
-    pytestdev: git+https://github.com/pytest-dev/pytest#egg=pytest
+    pytest70: pytest==7.0.*
+    pytest71: pytest==7.1.*
+    pytest72: pytest==7.2.*
+    pytest73: pytest==7.3.*
+    pytest74: pytest==7.4.*
+    devdeps: git+https://github.com/pytest-dev/pytest#egg=pytest
+    devdeps: numpy>=0.0.dev0
+    devdeps: pandas>=0.0.dev0
+    devdeps: astropy>=0.0.dev0
 extras =
     test
 commands =
+    # Force numpy-dev after something in the stack downgrades it
+    devdeps: python -m pip install --pre --upgrade --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
     pip freeze
     pytest {toxinidir}/tests {posargs}
     pytest {toxinidir}/tests --arraydiff {posargs}
 
 [testenv:codestyle]
 skip_install = true
+changedir = {toxinidir}
 description = check code style, e.g. with flake8
 deps = flake8
 commands = flake8 pytest_arraydiff --count