summaryrefslogtreecommitdiff
path: root/silx/opencl/codec/test/test_byte_offset.py
diff options
context:
space:
mode:
Diffstat (limited to 'silx/opencl/codec/test/test_byte_offset.py')
-rw-r--r--silx/opencl/codec/test/test_byte_offset.py317
1 files changed, 317 insertions, 0 deletions
diff --git a/silx/opencl/codec/test/test_byte_offset.py b/silx/opencl/codec/test/test_byte_offset.py
new file mode 100644
index 0000000..2bfa1d3
--- /dev/null
+++ b/silx/opencl/codec/test/test_byte_offset.py
@@ -0,0 +1,317 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Project: Byte-offset decompression in OpenCL
+# https://github.com/silx-kit/silx
+#
+# Copyright (C) 2013-2018 European Synchrotron Radiation Facility,
+# Grenoble, France
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+"""
+Test suite for byte-offset decompression
+"""
+
+from __future__ import division, print_function
+
+__authors__ = ["Jérôme Kieffer"]
+__contact__ = "jerome.kieffer@esrf.eu"
+__license__ = "MIT"
+__copyright__ = "2013 European Synchrotron Radiation Facility, Grenoble, France"
+__date__ = "10/11/2017"
+
+import sys
+import time
+import logging
+import numpy
+from silx.opencl.common import ocl, pyopencl
+from silx.opencl.codec import byte_offset
+try:
+ import fabio
+except ImportError:
+ fabio = None
+import unittest
+logger = logging.getLogger(__name__)
+
+
+@unittest.skipUnless(ocl and fabio and pyopencl,
+ "PyOpenCl or fabio is missing")
+class TestByteOffset(unittest.TestCase):
+
+ @staticmethod
+ def _create_test_data(shape, nexcept, lam=200):
+ """Create test (image, compressed stream) pair.
+
+ :param shape: Shape of test image
+ :param int nexcept: Number of exceptions in the image
+ :param lam: Expectation of interval argument for numpy.random.poisson
+ :return: (reference image array, compressed stream)
+ """
+ size = numpy.prod(shape)
+ ref = numpy.random.poisson(lam, numpy.prod(shape))
+ exception_loc = numpy.random.randint(0, size, size=nexcept)
+ exception_value = numpy.random.randint(0, 1000000, size=nexcept)
+ ref[exception_loc] = exception_value
+ ref.shape = shape
+
+ raw = fabio.compression.compByteOffset(ref)
+ return ref, raw
+
+ def test_decompress(self):
+ """
+ tests the byte offset decompression on GPU
+ """
+ ref, raw = self._create_test_data(shape=(91, 97), nexcept=229)
+ #ref, raw = self._create_test_data(shape=(7, 9), nexcept=0)
+
+ size = numpy.prod(ref.shape)
+
+ try:
+ bo = byte_offset.ByteOffset(raw_size=len(raw), dec_size=size, profile=True)
+ except (RuntimeError, pyopencl.RuntimeError) as err:
+ logger.warning(err)
+ if sys.platform == "darwin":
+ raise unittest.SkipTest("Byte-offset decompression is known to be buggy on MacOS-CPU")
+ else:
+ raise err
+ print(bo.block_size)
+
+ t0 = time.time()
+ res_cy = fabio.compression.decByteOffset(raw)
+ t1 = time.time()
+ res_cl = bo.decode(raw)
+ t2 = time.time()
+ delta_cy = abs(ref.ravel() - res_cy).max()
+ delta_cl = abs(ref.ravel() - res_cl.get()).max()
+
+ logger.debug("Global execution time: fabio %.3fms, OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0),
+ 1000.0 * (t2 - t1))
+ bo.log_profile()
+ #print(ref)
+ #print(res_cl.get())
+ self.assertEqual(delta_cy, 0, "Checks fabio works")
+ self.assertEqual(delta_cl, 0, "Checks opencl works")
+
+ def test_many_decompress(self, ntest=10):
+ """
+ tests the byte offset decompression on GPU, many images to ensure there
+ is not leaking in memory
+ """
+ shape = (991, 997)
+ size = numpy.prod(shape)
+ ref, raw = self._create_test_data(shape=shape, nexcept=0, lam=100)
+
+ try:
+ bo = byte_offset.ByteOffset(len(raw), size, profile=False)
+ except (RuntimeError, pyopencl.RuntimeError) as err:
+ logger.warning(err)
+ if sys.platform == "darwin":
+ raise unittest.SkipTest("Byte-offset decompression is known to be buggy on MacOS-CPU")
+ else:
+ raise err
+ t0 = time.time()
+ res_cy = fabio.compression.decByteOffset(raw)
+ t1 = time.time()
+ res_cl = bo(raw)
+ t2 = time.time()
+ delta_cy = abs(ref.ravel() - res_cy).max()
+ delta_cl = abs(ref.ravel() - res_cl.get()).max()
+ self.assertEqual(delta_cy, 0, "Checks fabio works")
+ self.assertEqual(delta_cl, 0, "Checks opencl works")
+ logger.debug("Global execution time: fabio %.3fms, OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0),
+ 1000.0 * (t2 - t1))
+
+ for i in range(ntest):
+ ref, raw = self._create_test_data(shape=shape, nexcept=2729, lam=200)
+
+ t0 = time.time()
+ res_cy = fabio.compression.decByteOffset(raw)
+ t1 = time.time()
+ res_cl = bo(raw)
+ t2 = time.time()
+ delta_cy = abs(ref.ravel() - res_cy).max()
+ delta_cl = abs(ref.ravel() - res_cl.get()).max()
+ self.assertEqual(delta_cy, 0, "Checks fabio works #%i" % i)
+ self.assertEqual(delta_cl, 0, "Checks opencl works #%i" % i)
+
+ logger.debug("Global execution time: fabio %.3fms, OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0),
+ 1000.0 * (t2 - t1))
+
+ def test_encode(self):
+ """Test byte offset compression"""
+ ref, raw = self._create_test_data(shape=(2713, 2719), nexcept=2729)
+
+ try:
+ bo = byte_offset.ByteOffset(len(raw), ref.size, profile=True)
+ except (RuntimeError, pyopencl.RuntimeError) as err:
+ logger.warning(err)
+ raise err
+
+ t0 = time.time()
+ compressed_array = bo.encode(ref)
+ t1 = time.time()
+
+ compressed_stream = compressed_array.get().tostring()
+ self.assertEqual(raw, compressed_stream)
+
+ logger.debug("Global execution time: OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0))
+ bo.log_profile()
+
+ def test_encode_to_array(self):
+ """Test byte offset compression while providing an out array"""
+
+ ref, raw = self._create_test_data(shape=(2713, 2719), nexcept=2729)
+
+ try:
+ bo = byte_offset.ByteOffset(profile=True)
+ except (RuntimeError, pyopencl.RuntimeError) as err:
+ logger.warning(err)
+ raise err
+ # Test with out buffer too small
+ out = pyopencl.array.empty(bo.queue, (10,), numpy.int8)
+ with self.assertRaises(ValueError):
+ bo.encode(ref, out)
+
+ # Test with out buffer too big
+ out = pyopencl.array.empty(bo.queue, (len(raw) + 10,), numpy.int8)
+
+ compressed_array = bo.encode(ref, out)
+
+ # Get size from returned array
+ compressed_size = compressed_array.size
+ self.assertEqual(compressed_size, len(raw))
+
+ # Get data from out array, read it from bo object queue
+ out_bo_queue = out.with_queue(bo.queue)
+ compressed_stream = out_bo_queue.get().tostring()[:compressed_size]
+ self.assertEqual(raw, compressed_stream)
+
+ def test_encode_to_bytes(self):
+ """Test byte offset compression to bytes"""
+ ref, raw = self._create_test_data(shape=(2713, 2719), nexcept=2729)
+
+ try:
+ bo = byte_offset.ByteOffset(profile=True)
+ except (RuntimeError, pyopencl.RuntimeError) as err:
+ logger.warning(err)
+ raise err
+
+ t0 = time.time()
+ res_fabio = fabio.compression.compByteOffset(ref)
+ t1 = time.time()
+ compressed_stream = bo.encode_to_bytes(ref)
+ t2 = time.time()
+
+ self.assertEqual(raw, compressed_stream)
+
+ logger.debug("Global execution time: fabio %.3fms, OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0),
+ 1000.0 * (t2 - t1))
+ bo.log_profile()
+
+ def test_encode_to_bytes_from_array(self):
+ """Test byte offset compression to bytes from a pyopencl array.
+ """
+ ref, raw = self._create_test_data(shape=(2713, 2719), nexcept=2729)
+
+ try:
+ bo = byte_offset.ByteOffset(profile=True)
+ except (RuntimeError, pyopencl.RuntimeError) as err:
+ logger.warning(err)
+ raise err
+
+ d_ref = pyopencl.array.to_device(
+ bo.queue, ref.astype(numpy.int32).ravel())
+
+ t0 = time.time()
+ res_fabio = fabio.compression.compByteOffset(ref)
+ t1 = time.time()
+ compressed_stream = bo.encode_to_bytes(d_ref)
+ t2 = time.time()
+
+ self.assertEqual(raw, compressed_stream)
+
+ logger.debug("Global execution time: fabio %.3fms, OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0),
+ 1000.0 * (t2 - t1))
+ bo.log_profile()
+
+ def test_many_encode(self, ntest=10):
+ """Test byte offset compression with many image"""
+ shape = (991, 997)
+ ref, raw = self._create_test_data(shape=shape, nexcept=0, lam=100)
+
+ try:
+ bo = byte_offset.ByteOffset(profile=False)
+ except (RuntimeError, pyopencl.RuntimeError) as err:
+ logger.warning(err)
+ raise err
+
+ bo_durations = []
+
+ t0 = time.time()
+ res_fabio = fabio.compression.compByteOffset(ref)
+ t1 = time.time()
+ compressed_stream = bo.encode_to_bytes(ref)
+ t2 = time.time()
+ bo_durations.append(1000.0 * (t2 - t1))
+
+ self.assertEqual(raw, compressed_stream)
+ logger.debug("Global execution time: fabio %.3fms, OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0),
+ 1000.0 * (t2 - t1))
+
+ for i in range(ntest):
+ ref, raw = self._create_test_data(shape=shape, nexcept=2729, lam=200)
+
+ t0 = time.time()
+ res_fabio = fabio.compression.compByteOffset(ref)
+ t1 = time.time()
+ compressed_stream = bo.encode_to_bytes(ref)
+ t2 = time.time()
+ bo_durations.append(1000.0 * (t2 - t1))
+
+ self.assertEqual(raw, compressed_stream)
+ logger.debug("Global execution time: fabio %.3fms, OpenCL: %.3fms.",
+ 1000.0 * (t1 - t0),
+ 1000.0 * (t2 - t1))
+
+ logger.debug("OpenCL execution time: Mean: %fms, Min: %fms, Max: %fms",
+ numpy.mean(bo_durations),
+ numpy.min(bo_durations),
+ numpy.max(bo_durations))
+
+
+def suite():
+ test_suite = unittest.TestSuite()
+ test_suite.addTest(TestByteOffset("test_decompress"))
+ test_suite.addTest(TestByteOffset("test_many_decompress"))
+ test_suite.addTest(TestByteOffset("test_encode"))
+ test_suite.addTest(TestByteOffset("test_encode_to_array"))
+ test_suite.addTest(TestByteOffset("test_encode_to_bytes"))
+ test_suite.addTest(TestByteOffset("test_encode_to_bytes_from_array"))
+ test_suite.addTest(TestByteOffset("test_many_encode"))
+ return test_suite