diff options
author | Onderwaater <onderwaa@esrf.fr> | 2015-12-14 09:24:29 +0100 |
---|---|---|
committer | Onderwaater <onderwaa@esrf.fr> | 2015-12-14 09:24:29 +0100 |
commit | 9bea7cc3ffa0ab6263054c39fa2db7b413a191c0 (patch) | |
tree | 0ebe81446225f4bb16b709d5e4e61102d9388dac /binoculars | |
parent | cf74ea2a392e1904efc78774caa57d8f63a6656a (diff) |
python3 support + old style connections removal + bugfix in limits
Diffstat (limited to 'binoculars')
-rw-r--r-- | binoculars/__init__.py | 6 | ||||
-rw-r--r-- | binoculars/backend.py | 2 | ||||
-rw-r--r-- | binoculars/backends/bm25.py | 5 | ||||
-rw-r--r-- | binoculars/backends/bm32.py | 14 | ||||
-rw-r--r-- | binoculars/backends/example.py | 2 | ||||
-rw-r--r-- | binoculars/backends/id03.py | 26 | ||||
-rw-r--r-- | binoculars/backends/id03_xu.py | 20 | ||||
-rw-r--r-- | binoculars/backends/sixs.py | 8 | ||||
-rwxr-xr-x | binoculars/dispatcher.py | 17 | ||||
-rw-r--r-- | binoculars/fit.py | 34 | ||||
-rwxr-xr-x | binoculars/main.py | 3 | ||||
-rwxr-xr-x | binoculars/space.py | 33 | ||||
-rwxr-xr-x | binoculars/util.py | 74 |
13 files changed, 154 insertions, 90 deletions
diff --git a/binoculars/__init__.py b/binoculars/__init__.py index 4ed0103..82734bb 100644 --- a/binoculars/__init__.py +++ b/binoculars/__init__.py @@ -1,9 +1,9 @@ +from __future__ import print_function, with_statement, division + import os import sys # for scripted useage - - def run(args): '''Parameters args: string @@ -205,7 +205,7 @@ def fitspace(space, function, guess=None): Examples: >>> fit = binoculars.fitspace(space, 'lorentzian') - >>> print fit.summary + >>> print(fit.summary) I: 1.081e-07 +/- inf loc: 0.3703 +/- inf gamma: 0.02383 +/- inf diff --git a/binoculars/backend.py b/binoculars/backend.py index c2e5d5a..7c54a3e 100644 --- a/binoculars/backend.py +++ b/binoculars/backend.py @@ -5,7 +5,7 @@ class ProjectionBase(util.ConfigurableObject): def parse_config(self, config): super(ProjectionBase, self).parse_config(config) res = config.pop('resolution') # or just give 1 number for all dimensions - self.config.limits = util.parse_pairs(config.pop('limits', None)) # Optional, set the limits of the space object in projected coordinates. Syntax is same as numpy e.g. '0.3:-0.6, -1:5, :' + self.config.limits = util.parse_pairs(config.pop('limits', None)) # Optional, set the limits of the space object in projected coordinates. Syntax is same as numpy e.g. 'limits = [:0,-1:,:], [0:,:-1,:], [:0,:-1,:], [0:,-1:,:]' labels = self.get_axis_labels() if not self.config.limits is None: for lim in self.config.limits: diff --git a/binoculars/backends/bm25.py b/binoculars/backends/bm25.py index d9e8adc..06d3bbf 100644 --- a/binoculars/backends/bm25.py +++ b/binoculars/backends/bm25.py @@ -21,7 +21,6 @@ author: Dominik Kriegner (dominik.kriegner@gmail.com) import sys import os -import itertools import glob import numpy import xrayutilities as xu @@ -177,7 +176,7 @@ class EH2SCD(EDFInput): pwidth2=self.config.pixelsize[0], distance=1e-10, roi=roi) - print('{:>20} {:>9} {:>10} {:>9} {:>9} {:>9}'.format(' ', 'Mu', 'Theta', 'CCD_Y', 'CCD_X', 'CCD_Z')) + print(('{:>20} {:>9} {:>10} {:>9} {:>9} {:>9}'.format(' ', 'Mu', 'Theta', 'CCD_Y', 'CCD_X', 'CCD_Z'))) def process_image(self, image): # motor positions @@ -205,7 +204,7 @@ class EH2SCD(EDFInput): # normalization data = image.data / mon / transm - print('{:>20} {:9.4f} {:10.4f} {:9.1f} {:9.1f} {:9.1f}'.format(os.path.split(image.filename)[-1] ,mu, th, cty, ctx, ctz)) + print(('{:>20} {:9.4f} {:10.4f} {:9.1f} {:9.1f} {:9.1f}'.format(os.path.split(image.filename)[-1] ,mu, th, cty, ctx, ctz))) # masking intensity = self.apply_mask(data, self.config.xmask, self.config.ymask) diff --git a/binoculars/backends/bm32.py b/binoculars/backends/bm32.py index cf03d3f..993ad5c 100644 --- a/binoculars/backends/bm32.py +++ b/binoculars/backends/bm32.py @@ -1,10 +1,16 @@ import sys import os -import itertools import glob import numpy import time +#python3 support +PY3 = sys.version_info > (3,) +if PY3: + pass +else: + from itertools import izip as zip + try: from PyMca import specfilewrapper, EdfFile, SixCircle, specfile except ImportError: @@ -192,7 +198,7 @@ class BM32Input(backend.InputBase): pointparams = self.get_point_params(scan, job.firstpoint, job.lastpoint) # 2D array of diffractometer angles + mon + transm images = self.get_images(scan, job.firstpoint, job.lastpoint) # iterator! - for pp, image in itertools.izip(pointparams, images): + for pp, image in zip(pointparams, images): yield self.process_image(scanparams, pp, image) util.statuseol() except Exception as exc: @@ -277,7 +283,7 @@ class BM32Input(backend.InputBase): uccdtagline = scan.header('M')[0].split()[-1] UCCD = os.path.dirname(uccdtagline).split(os.sep) except: - print 'warning: UCCD tag not found, use imagefolder for proper file specification' + print('warning: UCCD tag not found, use imagefolder for proper file specification') UCCD = [] pattern = self._get_pattern(UCCD) matches = self.find_edfs(pattern) @@ -377,7 +383,7 @@ class EH1(BM32Input): def get_point_params(self, scan, first, last): sl = slice(first, last+1) - DEL, OME, ALF, BET, CHI, PHI, MON, TRANSM = range(8) + DEL, OME, ALF, BET, CHI, PHI, MON, TRANSM = list(range(8)) params = numpy.zeros((last - first + 1, 8)) # gamma delta theta chi phi mu mon transm params[:, CHI] = 0 #scan.motorpos('CHI') params[:, PHI] = 0 #scan.motorpos('PHI') diff --git a/binoculars/backends/example.py b/binoculars/backends/example.py index c763eb8..d79bb52 100644 --- a/binoculars/backends/example.py +++ b/binoculars/backends/example.py @@ -78,7 +78,7 @@ class Input(backend.InputBase): aai = numpy.linspace(0, numpy.random.random() * 20, 100) aomega = numpy.linspace(0, numpy.random.random() * 20, 100) for af, delta, ai, omega in zip(aaf, adelta, aai, aomega): - print 'af: {0}, delta: {1}, ai: {2}, omega: {3}'.format(af, delta, ai, omega) + print('af: {0}, delta: {1}, ai: {2}, omega: {3}'.format(af, delta, ai, omega)) # caculating the angles per pixel. The values specified in the configuration file # can be used for calculating these values diff --git a/binoculars/backends/id03.py b/binoculars/backends/id03.py index 9389517..a388348 100644 --- a/binoculars/backends/id03.py +++ b/binoculars/backends/id03.py @@ -5,12 +5,18 @@ import glob import numpy import time +#python3 support +PY3 = sys.version_info > (3,) +if PY3: + pass +else: + from itertools import izip as zip + try: from PyMca import specfilewrapper, EdfFile, SixCircle, specfile except ImportError: from PyMca5.PyMca import specfilewrapper, EdfFile, SixCircle, specfile - from .. import backend, errors, util @@ -290,7 +296,7 @@ class ID03Input(backend.InputBase): yield backend.Job(scan=scanno, firstpoint=firstpoint+s.start, lastpoint=firstpoint+s.stop-1, weight=s.stop-s.start) else: # scanlength is unknown step = int(self.config.target_weight / 1.4) - for start, stop in itertools.izip(itertools.count(0, step), itertools.count(step, step)): + for start, stop in zip(itertools.count(0, step), itertools.count(step, step)): if self.wait_for_points(scanno, stop, timeout=self.config.timeout): stop = self.get_scan(scanno).lines() yield backend.Job(scan=scanno, firstpoint=start, lastpoint=stop-1, weight=stop-start) @@ -307,7 +313,7 @@ class ID03Input(backend.InputBase): pointparams = self.get_point_params(scan, job.firstpoint, job.lastpoint) # 2D array of diffractometer angles + mon + transm images = self.get_images(scan, job.firstpoint, job.lastpoint) # iterator! - for pp, image in itertools.izip(pointparams, images): + for pp, image in zip(pointparams, images): yield self.process_image(scanparams, pp, image) util.statuseol() except Exception as exc: @@ -418,7 +424,7 @@ class ID03Input(backend.InputBase): filename = os.path.basename(file).split('.')[0] scan, point, image = filename.split('_')[-3:] scan, point, image = int(scan), int(point), int(image) - if scan == scanno and point not in ret.keys(): + if scan == scanno and point not in list(ret.keys()): ret[point] = file except ValueError: continue @@ -480,7 +486,7 @@ class ID03Input(backend.InputBase): uccdtagline = scanheaderC[0] UCCD = os.path.split(uccdtagline.split()[-1]) except: - print 'warning: UCCD tag not found, use imagefolder for proper file specification' + print('warning: UCCD tag not found, use imagefolder for proper file specification') UCCD = [] pattern = self._get_pattern(UCCD) matches = self.find_edfs(pattern, zapscanno) @@ -499,7 +505,7 @@ class ID03Input(backend.InputBase): uccdtagline = scan.header('UCCD')[0] UCCD = os.path.split(os.path.dirname(uccdtagline.split()[-1])) except: - print 'warning: UCCD tag not found, use imagefolder for proper file specification' + print('warning: UCCD tag not found, use imagefolder for proper file specification') UCCD = [] pattern = self._get_pattern(UCCD) matches = self.find_edfs(pattern, scan.number()) @@ -595,7 +601,7 @@ class EH1(ID03Input): def get_point_params(self, scan, first, last): sl = slice(first, last+1) - GAM, DEL, TH, CHI, PHI, MU, MON, TRANSM, HRX, HRY = range(10) + GAM, DEL, TH, CHI, PHI, MU, MON, TRANSM, HRX, HRY = list(range(10)) params = numpy.zeros((last - first + 1, 10)) # gamma delta theta chi phi mu mon transm params[:, CHI] = scan.motorpos('Chi') params[:, PHI] = scan.motorpos('Phi') @@ -712,7 +718,7 @@ class EH2(ID03Input): def get_point_params(self, scan, first, last): sl = slice(first, last+1) - GAM, DEL, TH, CHI, PHI, MU, MON, TRANSM = range(8) + GAM, DEL, TH, CHI, PHI, MU, MON, TRANSM = list(range(8)) params = numpy.zeros((last - first + 1, 8)) # gamma delta theta chi phi mu mon transm params[:, CHI] = scan.motorpos('Chi') params[:, PHI] = scan.motorpos('Phi') @@ -808,7 +814,7 @@ class GisaxsDetector(ID03Input): def get_point_params(self, scan, first, last): sl = slice(first, last+1) - CCDY, CCDZ, TH, CHI, PHI, MU, MON, TRANSM = range(8) + CCDY, CCDZ, TH, CHI, PHI, MU, MON, TRANSM = list(range(8)) params = numpy.zeros((last - first + 1, 8)) # gamma delta theta chi phi mu mon transm params[:, CHI] = scan.motorpos('Chi') params[:, PHI] = scan.motorpos('Phi') @@ -834,7 +840,7 @@ class GisaxsDetector(ID03Input): filename = os.path.basename(file).split('.')[0] scan, point = filename.split('_')[-2:] scan, point = int(scan), int(point) - if scan == scanno and point not in ret.keys(): + if scan == scanno and point not in list(ret.keys()): ret[point] = file except ValueError: continue diff --git a/binoculars/backends/id03_xu.py b/binoculars/backends/id03_xu.py index d8c3fe0..a10f617 100644 --- a/binoculars/backends/id03_xu.py +++ b/binoculars/backends/id03_xu.py @@ -13,13 +13,19 @@ author: Dominik Kriegner (dominik.kriegner@gmail.com) import sys import os -import itertools import glob import numpy import xrayutilities as xu from PyMca import specfile +#python3 support +PY3 = sys.version_info > (3,) +if PY3: + pass +else: + import itertools import izip as zip + try: from PyMca import specfilewrapper, EdfFile except ImportError: @@ -87,7 +93,7 @@ class ID03Input(backend.InputBase): pointparams = self.get_point_params(scan, job.firstpoint, job.lastpoint) # 1D array of diffractometer angles + mon + transm images = self.get_images(scan, job.firstpoint, job.lastpoint) # iterator! - for pp, image in itertools.izip(pointparams, images): + for pp, image in zip(pointparams, images): yield self.process_image(scanparams, pp, image) def parse_config(self, config): @@ -125,7 +131,7 @@ class ID03Input(backend.InputBase): filename = os.path.basename(file).split('.')[0] scan, point, image = filename.split('_')[-3:] scan, point, image = int(scan), int(point), int(image) - if scan == scanno and point not in ret.keys(): + if scan == scanno and point not in list(ret.keys()): ret[point] = file except ValueError: continue @@ -149,7 +155,7 @@ class ID03Input(backend.InputBase): uccdtagline = scan.header('UCCD')[0] UCCD = os.path.split(os.path.dirname(uccdtagline.split()[-1])) except: - print 'warning: UCCD tag not found, use imagefolder for proper file specification' + print('warning: UCCD tag not found, use imagefolder for proper file specification') UCCD = [] pattern = self._get_pattern(UCCD) matches = self.find_edfs(pattern, scan.number()) @@ -211,13 +217,13 @@ class EH2(ID03Input): # distance sdd-600 corresponds to distance of the detector chip from # the gamR rotation axis (rest is handled by the translations ty and # gamT (along z)) - print('{:>9} {:>10} {:>9} {:>9}'.format('Mu', 'Theta', 'Delta', 'Gamma')) + print(('{:>9} {:>10} {:>9} {:>9}'.format('Mu', 'Theta', 'Delta', 'Gamma'))) def process_image(self, scanparams, pointparams, image): mu, theta, chi, phi, delta, gamma, mon, transm = pointparams wavelength, UB = scanparams data = image / mon / transm - print('{:9.4f} {:10.4f} {:9.4f} {:9.4f}'.format(mu, theta, delta, gamma)) + print(('{:9.4f} {:10.4f} {:9.4f} {:9.4f}'.format(mu, theta, delta, gamma))) # recalculate detector translation (which should be saved!) gamT = self.ty * numpy.tan(numpy.radians(gamma)) @@ -233,7 +239,7 @@ class EH2(ID03Input): def get_point_params(self, scan, first, last): sl = slice(first, last+1) - MU, TH, CHI, PHI, DEL, GAM, MON, TRANSM = range(8) + MU, TH, CHI, PHI, DEL, GAM, MON, TRANSM = list(range(8)) params = numpy.zeros((last - first + 1, 8)) # Mu, Theta, Chi, Phi, Delta, Gamma, MON, transm params[:, CHI] = scan.motorpos('Chi') diff --git a/binoculars/backends/sixs.py b/binoculars/backends/sixs.py index 57fff05..4e47166 100644 --- a/binoculars/backends/sixs.py +++ b/binoculars/backends/sixs.py @@ -41,6 +41,10 @@ from networkx import DiGraph, dijkstra_path from .. import backend, errors, util +if PY3: + from functools import reduce +else: + from itertools import izip as zip class realspace(backend.ProjectionBase): # scalars: mu, theta, [chi, phi, "omitted"] delta, gamR, gamT, ty, wavelength @@ -269,7 +273,7 @@ def dataframes(hfile, data_path=None): } # now instantiate the pytables objects - for key, value in data_path.iteritems(): + for key, value in data_path.items(): child = scan_data._f_get_child(value) dataframe[key] = child @@ -311,7 +315,7 @@ def rotation_axes(graph, nodes): def zip_with(f, *coll): - return itertools.starmap(f, itertools.izip(*coll)) + return itertools.starmap(f, zip(*coll)) def rotation_matrix(values, axes): diff --git a/binoculars/dispatcher.py b/binoculars/dispatcher.py index 83b4ee8..7a05043 100755 --- a/binoculars/dispatcher.py +++ b/binoculars/dispatcher.py @@ -1,3 +1,4 @@ +import sys import os import time import itertools @@ -6,6 +7,8 @@ import multiprocessing from . import util, errors, space +#python3 support +PY3 = sys.version_info > (3,) class Destination(object): type = filename = overwrite = value = config = limits = None @@ -151,14 +154,14 @@ class Local(ReentrantBase): self.config.ncores = multiprocessing.cpu_count() def process_jobs(self, jobs): - if self.config.ncores == 1: # note: SingleCore will be marginally faster - imap = itertools.imap + if self.config.ncores == 1 and not PY3: # note: SingleCore will be marginally faster + map = itertools.imap else: pool = multiprocessing.Pool(self.config.ncores) - imap = pool.imap_unordered + map = pool.imap_unordered configs = (self.prepare_config(job) for job in jobs) - for result in imap(self.main.get_reentrant(), configs): + for result in map(self.main.get_reentrant(), configs): yield result def sum(self, results): @@ -311,7 +314,7 @@ class Oar(ReentrantBase): try: os.remove(f) except Exception as e: - print "unable to remove {0}: {1}".format(f, e) + print("unable to remove {0}: {1}".format(f, e)) errorfn = [] @@ -323,7 +326,7 @@ class Oar(ReentrantBase): errormsg = fp.read() if len(errormsg) > 0: errorfn.append(errorfilename) - print 'Critical error: OAR Job {0} failed with the following error: \n{1}'.format(jobid, errormsg) + print('Critical error: OAR Job {0} failed with the following error: \n{1}'.format(jobid, errormsg)) if len(errorfn) > 0: - print 'Warning! {0} job(s) failed. See above for the details or the error log files: {1}'.format(len(errorfn), ', '.join(errorfn)) + print('Warning! {0} job(s) failed. See above for the details or the error log files: {1}'.format(len(errorfn), ', '.join(errorfn))) diff --git a/binoculars/fit.py b/binoculars/fit.py index 028b8e5..a9333ad 100644 --- a/binoculars/fit.py +++ b/binoculars/fit.py @@ -133,7 +133,7 @@ def rot3d(x, y, z, th, ph): def get_class_by_name(name): options = {} - for k, v in globals().iteritems(): + for k, v in globals().items(): if isinstance(v, type) and issubclass(v, FitBase): options[k.lower()] = v if name.lower() in options: @@ -145,7 +145,9 @@ def get_class_by_name(name): # fitting functions class Lorentzian1D(PeakFitBase): @staticmethod - def func((x, ), (I, loc, gamma, slope, offset)): + def func(xxx_todo_changeme, xxx_todo_changeme1): + (x, ) = xxx_todo_changeme + (I, loc, gamma, slope, offset) = xxx_todo_changeme1 return I / ((x - loc)**2 + gamma**2) + offset + x * slope def set_guess(self, maximum, argmax, linparams): @@ -155,7 +157,9 @@ class Lorentzian1D(PeakFitBase): class Lorentzian1DNoBkg(PeakFitBase): @staticmethod - def func((x, ), (I, loc, gamma)): + def func(xxx_todo_changeme2, xxx_todo_changeme3): + (x, ) = xxx_todo_changeme2 + (I, loc, gamma) = xxx_todo_changeme3 return I / ((x - loc)**2 + gamma**2) def set_guess(self, maximum, argmax, linparams): @@ -165,7 +169,9 @@ class Lorentzian1DNoBkg(PeakFitBase): class PolarLorentzian2Dnobkg(PeakFitBase): @staticmethod - def func((x, y), (I, loc0, loc1, gamma0, gamma1, th)): + def func(xxx_todo_changeme4, xxx_todo_changeme5): + (x, y) = xxx_todo_changeme4 + (I, loc0, loc1, gamma0, gamma1, th) = xxx_todo_changeme5 a, b = tuple(grid - center for grid, center in zip(rot2d(x, y, th), rot2d(loc0, loc1, th))) return (I / (1 + (a / gamma0)**2 + (b / gamma1)**2)) @@ -177,7 +183,9 @@ class PolarLorentzian2Dnobkg(PeakFitBase): class PolarLorentzian2D(PeakFitBase): @staticmethod - def func((x, y), (I, loc0, loc1, gamma0, gamma1, th, slope1, slope2, offset)): + def func(xxx_todo_changeme6, xxx_todo_changeme7): + (x, y) = xxx_todo_changeme6 + (I, loc0, loc1, gamma0, gamma1, th, slope1, slope2, offset) = xxx_todo_changeme7 a, b = tuple(grid - center for grid, center in zip(rot2d(x, y, th), rot2d(loc0, loc1, th))) return (I / (1 + (a / gamma0)**2 + (b / gamma1)**2) + x * slope1 + y * slope2 + offset) @@ -192,7 +200,9 @@ class PolarLorentzian2D(PeakFitBase): class Lorentzian2D(PeakFitBase): @staticmethod - def func((x, y), (I, loc0, loc1, gamma0, gamma1, th, slope1, slope2, offset)): + def func(xxx_todo_changeme8, xxx_todo_changeme9): + (x, y) = xxx_todo_changeme8 + (I, loc0, loc1, gamma0, gamma1, th, slope1, slope2, offset) = xxx_todo_changeme9 a, b = tuple(grid - center for grid, center in zip(rot2d(x, y, th), rot2d(loc0, loc1, th))) return (I / (1 + (a/gamma0)**2) * 1 / (1 + (b/gamma1)**2) + x * slope1 + y * slope2 + offset) @@ -204,7 +214,9 @@ class Lorentzian2D(PeakFitBase): class Lorentzian2Dnobkg(PeakFitBase): @staticmethod - def func((x, y), (I, loc0, loc1, gamma0, gamma1, th)): + def func(xxx_todo_changeme10, xxx_todo_changeme11): + (x, y) = xxx_todo_changeme10 + (I, loc0, loc1, gamma0, gamma1, th) = xxx_todo_changeme11 a, b = tuple(grid - center for grid, center in zip(rot2d(x, y, th), rot2d(loc0, loc1, th))) return (I / (1 + (a/gamma0)**2) * 1 / (1 + (b/gamma1)**2)) @@ -220,13 +232,17 @@ class Lorentzian(AutoDimensionFit): class Gaussian1D(PeakFitBase): @staticmethod - def func((x,), (loc, I, sigma, offset, slope)): + def func(xxx_todo_changeme12, xxx_todo_changeme13): + (x,) = xxx_todo_changeme12 + (loc, I, sigma, offset, slope) = xxx_todo_changeme13 return I * numpy.exp(-((x-loc)/sigma)**2/2) + offset + x * slope class Voigt1D(PeakFitBase): @staticmethod - def func((x, ), (I, loc, sigma, gamma, slope, offset)): + def func(xxx_todo_changeme14, xxx_todo_changeme15): + (x, ) = xxx_todo_changeme14 + (I, loc, sigma, gamma, slope, offset) = xxx_todo_changeme15 z = (x - loc + numpy.complex(0, gamma)) / (sigma * numpy.sqrt(2)) return I * numpy.real(scipy.special.wofz(z))/(sigma * numpy.sqrt(2 * numpy.pi)) + offset + x * slope diff --git a/binoculars/main.py b/binoculars/main.py index 188427b..6c80f1b 100755 --- a/binoculars/main.py +++ b/binoculars/main.py @@ -22,7 +22,8 @@ def parse_commandline_config_option(s): return section, option, value -def multiprocessing_main((config, command)): # note the double parenthesis for map() convenience +def multiprocessing_main(xxx_todo_changeme): # note the double parenthesis for map() convenience + (config, command) = xxx_todo_changeme Main.from_object(config, command) return config.dispatcher.destination.retrieve() diff --git a/binoculars/space.py b/binoculars/space.py index cc9138f..de501d3 100755 --- a/binoculars/space.py +++ b/binoculars/space.py @@ -1,10 +1,20 @@ -import itertools +from __future__ import unicode_literals + import numbers import numpy import h5py +import sys +from itertools import chain from . import util, errors +#python3 support +PY3 = sys.version_info > (3,) +if PY3: + from functools import reduce + basestring = (str,bytes) +else: + from itertools import izip as zip def silence_numpy_errors(): """Silence numpy warnings about zero division. Normal usage of Space() will trigger these warnings.""" @@ -13,7 +23,7 @@ def silence_numpy_errors(): def sum_onto(a, axis): """Numpy convenience. Project all dimensions of an array onto an axis, i.e. apply sum() to all axes except the one given.""" - for i in reversed(range(len(a.shape))): + for i in reversed(list(range(len(a.shape)))): if axis != i: a = a.sum(axis=i) return a @@ -56,7 +66,7 @@ class Axis(object): return self.imax - self.imin + 1 def __iter__(self): - return iter(self[index] for index in xrange(len(self))) + return iter(self[index] for index in range(len(self))) def __getitem__(self, key): if isinstance(key, slice): @@ -568,7 +578,7 @@ class Space(object): weights = self.contributions # get rid of invalid coords - valid = reduce(numpy.bitwise_and, intertools.chain((numpy.isfinite(t) for t in transcoords)), (weights > 0, )) + valid = reduce(numpy.bitwise_and, chain((numpy.isfinite(t) for t in transcoords)), (weights > 0, )) transcoords = tuple(t[valid] for t in transcoords) return self.from_image(resolutions, labels, transcoords, intensity[valid], weights[valid]) @@ -582,7 +592,7 @@ class Space(object): if len(coordinates) != len(self.axes): raise ValueError('dimension mismatch between coordinates and axes') - intensity = numpy.nan_to_num(intensity).flatten() # invalids should be handeled by setting weight to 0, this ensures the weights can do that + intensity = numpy.nan_to_num(intensity).flatten() # invalids can be handeled by setting weight to 0, this ensures the weights can do that weights = weights.flatten() indices = numpy.array(tuple(ax.get_index(coord) for (ax, coord) in zip(self.axes, coordinates))) @@ -609,11 +619,11 @@ class Space(object): if limits is not None: invalid = numpy.zeros(intensity.shape).astype(numpy.bool) for coord, sl in zip(coordinates, limits): - if sl.start is None and sl.stop ia not None: + if sl.start is None and sl.stop is not None: invalid += coord > sl.stop elif sl.start is not None and sl.stop is None: invalid += coord < sl.start - elif sl.start is not None and sl.stop os not None: + elif sl.start is not None and sl.stop is not None: invalid += numpy.bitwise_or(coord < sl.start, coord > sl.stop) if numpy.all(invalid is True): @@ -657,6 +667,9 @@ class Space(object): if len(axes) != len(key): raise ValueError("dimensionality of 'key' does not match dimensionality of Space in HDF5 file {0}".format(file)) key = tuple(ax.get_index(k) for k, ax in zip(key, axes)) + for index, sl in enumerate(key): + if sl.start == sl.stop and sl.start is not None: + raise KeyError('key results in empty space') axes = tuple(ax[k] for k, ax in zip(key, axes) if isinstance(k, slice)) else: key = Ellipsis @@ -795,7 +808,7 @@ def sum(spaces): def verse_sum(verses): i = iter(M.spaces for M in verses) - return Multiverse(sum(spaces) for spaces in itertools.izip(*i)) + return Multiverse(sum(spaces) for spaces in zip(*i)) # hybrid sum() / __iadd__() @@ -864,7 +877,7 @@ def dstack(spaces, dindices, dlabel, dresolution): exprs.append('ones_like({0}) * {1}'.format(labels[0], dindex)) transformation = util.transformation_from_expressions(space, exprs) return space.transform_coordinates(resolutions, labels, transformation) - return sum(transform(space, dindex) for space, dindex in itertools.izip(spaces, dindices)) + return sum(transform(space, dindex) for space, dindex in zip(spaces, dindices)) def axis_offset(space, label, offset): @@ -897,5 +910,5 @@ def make_compatible(spaces): resmax = tuple(numpy.vstack(tuple(ax.res for ax in space.reorder(ax0).axes) for space in spaces).max(axis=0)) resmin = tuple(numpy.vstack(tuple(ax.res for ax in space.reorder(ax0).axes) for space in spaces).min(axis=0)) if not resmax == resmin: - print 'Warning: Not all spaces have the same resolution. Resolution will be changed to: {0}'.format(resmax) + print('Warning: Not all spaces have the same resolution. Resolution will be changed to: {0}'.format(resmax)) return tuple(space.reorder(ax0).rebin2(resmax) for space in spaces) diff --git a/binoculars/util.py b/binoculars/util.py index 30a09cd..4ef8377 100755 --- a/binoculars/util.py +++ b/binoculars/util.py @@ -1,9 +1,10 @@ +from __future__ import print_function, division + import os import sys import gzip import itertools import random -import cPickle as pickle import inspect import time import copy @@ -11,20 +12,29 @@ import numpy import contextlib import argparse import h5py -import ConfigParser import glob -import errors -import StringIO +from . import errors import struct import json import socket -import StringIO import binascii import re ### ARGUMENT HANDLING +#python3 support +PY3 = sys.version_info > (3,) +if PY3: + import pickle + import io + import configparser +else: + import StringIO as io + import Queue as queue + import cPickle as pickle + import ConfigParser as configparser + class OrderedOperation(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): oops = getattr(namespace, 'ordered_operations', []) @@ -115,7 +125,7 @@ def handle_ordered_operations(space, args, auto3to2=False): space = space.project(projectaxis) elif command == 'transform': - labels, resolutions, exprs = zip(*parse_transform_args(opts)) + labels, resolutions, exprs = list(zip(*parse_transform_args(opts))) transformation = transformation_from_expressions(space, exprs) info.append('transformed to {0}'.format(', '.join('{0} = {1}'.format(label, expr) for (label, expr) in zip(labels, exprs)))) space = space.transform_coordinates(resolutions, labels, transformation) @@ -188,17 +198,17 @@ def get_backends(): def get_projections(module): - import backend + from . import backend return get_base(module, backend.ProjectionBase) def get_inputs(module): - import backend + from . import backend return get_base(module, backend.InputBase) def get_dispatchers(): - import dispatcher + from . import dispatcher from inspect import isclass items = dir(dispatcher) @@ -238,7 +248,7 @@ def get_base(modname, base): def get_dispatcher_configkeys(classname): - import dispatcher + from . import dispatcher cls = getattr(dispatcher, classname) return get_configkeys(cls) @@ -290,7 +300,7 @@ def parse_configcode(line): def parse_range(r): if '-' in r: a, b = r.split('-') - return range(int(a), int(b)+1) + return list(range(int(a), int(b)+1)) elif r: return [int(r)] else: @@ -387,9 +397,9 @@ class MetaBase(object): for section in self.sections: section_dict = {} attr = getattr(self, section) - for key in attr.keys(): + for key in list(attr.keys()): if isinstance(attr[key], numpy.ndarray): # to be able to include numpy arrays in the serialisation - sio = StringIO.StringIO() + sio = io.StringIO() numpy.save(sio, attr[key]) sio.seek(0) section_dict[key] = binascii.b2a_hex(sio.read()) # hex codation is needed to let json work with the string @@ -402,12 +412,12 @@ class MetaBase(object): def fromserial(cls, s): obj = cls() data = json.loads(s) - for section in data.keys(): + for section in list(data.keys()): section_dict = data[section] - for key in section_dict.keys(): - if isinstance(section_dict[key], basestring): # find and replace all the numpy serialised objects + for key in list(section_dict.keys()): + if isinstance(section_dict[key], str): # find and replace all the numpy serialised objects if section_dict[key].startswith('934e554d505901004600'): # numpy marker - sio = StringIO.StringIO() + sio = io.StringIO() sio.write(binascii.a2b_hex(section_dict[key])) sio.seek(0) section_dict[key] = numpy.load(sio) @@ -416,10 +426,10 @@ class MetaBase(object): obj.sections.append(section) return obj -# a collection of metadata objects -class MetaData(object): + +class MetaData(object): # a collection of metadata objects def __init__(self): self.metas = [] @@ -441,7 +451,7 @@ class MetaData(object): @classmethod def fromfile(cls, filename): - if isinstance(filename, basestring): + if isinstance(filename, str): if not os.path.exists(filename): raise IOError('Error importing configuration file. filename {0} does not exist'.format(filename)) @@ -453,7 +463,7 @@ class MetaData(object): metadata = [] # when metadata is not present, proceed without Error for label in metadata: meta = MetaBase() - for section in metadata[label].keys(): + for section in list(metadata[label].keys()): group = metadata[label][section] setattr(meta, section, dict((key, group[key].value) for key in group)) meta.sections.append(section) @@ -464,12 +474,12 @@ class MetaData(object): with open_h5py(filename, 'w') as fp: metadata = fp.create_group('metadata') for meta in self.metas: - label = find_unused_label('metasection', metadata.keys()) + label = find_unused_label('metasection', list(metadata.keys())) metabase = metadata.create_group(label) for section in meta.sections: sectiongroup = metabase.create_group(section) s = getattr(meta, section) - for key in s.keys(): + for key in list(s.keys()): sectiongroup.create_dataset(key, data=s[key]) def __repr__(self): @@ -504,7 +514,7 @@ class ConfigFile(MetaBase): @classmethod def fromfile(cls, filename): - if isinstance(filename, basestring): + if isinstance(filename, str): if not os.path.exists(filename): raise IOError('Error importing configuration file. filename {0} does not exist'.format(filename)) @@ -513,7 +523,7 @@ class ConfigFile(MetaBase): try: config = fp['configuration'] if 'command' in config.attrs: - configobj.command = json.loads(config.attrs['command']) + configobj.command = json.loads(config.attrs['command'].decode('utf8')) for section in config: if isinstance(config[section], h5py._hl.group.Group): # new setattr(configobj, section, dict((key, config[section][key].value) for key in config[section])) @@ -528,7 +538,7 @@ class ConfigFile(MetaBase): if not os.path.exists(filename): raise IOError('Error importing configuration file. filename {0} does not exist'.format(filename)) - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() config.read(filename) for section, option, value in overrides: @@ -547,7 +557,7 @@ class ConfigFile(MetaBase): for section in self.sections: sectiongroup = conf.create_group(section) s = getattr(self, section) - for key in s.keys(): + for key in list(s.keys()): sectiongroup.create_dataset(key, data=s[key]) def totxtfile(self, filename): @@ -596,16 +606,16 @@ class ConfigurableObject(object): else: self.config = ConfigSection() try: - allkeys = config.keys() + allkeys = list(config.keys()) self.parse_config(config) except KeyError as exc: raise errors.ConfigError("Configuration option {0} is missing from the configuration file. Please specify this option in the configuration file".format(exc)) except Exception as exc: - missing = set(key for key in allkeys if key not in self.config.__dict__.keys()) - set(config.keys()) + missing = set(key for key in allkeys if key not in list(self.config.__dict__.keys())) - set(config.keys()) exc.args = errors.addmessage(exc.args, ". Unable to parse configuration option '{0}'. The error can quite likely be solved by modifying the option in the configuration file.".format(','.join(missing))) raise for k in config: - print 'warning: unrecognized configuration option {0} for {1}'.format(k, self.__class__.__name__) + print('warning: unrecognized configuration option {0} for {1}'.format(k, self.__class__.__name__)) self.config.class_ = self.__class__ def parse_config(self, config): @@ -892,7 +902,7 @@ def zpi_load(filename): def serialize(space, command): # first 48 bytes contain length of the message, whereby the first 8 give the length of the command, the second 8 the length of the configfile etc.. - message = StringIO.StringIO() + message = io.StringIO() message.write(struct.pack('QQQQQQ', 0, 0, 0, 0, 0, 0)) message.write(command) @@ -946,7 +956,7 @@ def socket_send(ip, port, mssg): def socket_recieve(RequestHandler): # pass one the handler to deal with incoming data def get_msg(length): - msg = StringIO.StringIO() + msg = io.StringIO() for packet in packet_slicer(length): p = RequestHandler.request.recv(packet, socket.MSG_WAITALL) # wait for full mssg msg.write(p) |