mupltiple updates

Backend cleanup, introduction of limits, 3d plotting, new hdf5 saving
structure,  delayed processing.

13 files changed, 502 insertions, 267 deletions

diff --git a/BINoculars/__init__.py b/BINoculars/__init__.py
index a79f096..dfbbec4 100644
--- a/BINoculars/__init__.py
+++ b/BINoculars/__init__.py
@@ -123,13 +123,17 @@ def plotspace(space, log=True, clipping=0.0, fit=None, norm=None, colorbar=True,
     import BINoculars.plot, BINoculars.space
 
     if isinstance(space, BINoculars.space.Space):
+        if space.dimension == 3:
+            from mpl_toolkits.mplot3d import Axes3D
+            ax = pyplot.gcf().gca(projection='3d')
+            return BINoculars.plot.plot(space, pyplot.gcf(), ax, log=log, clipping=clipping, fit=None, norm=norm, colorbar=colorbar, labels=labels, **plotopts)        
         if fit is not None and space.dimension == 2:
             ax = pyplot.gcf().add_subplot(121)
             BINoculars.plot.plot(space, pyplot.gcf(), ax, log=log, clipping=clipping, fit=None, norm=norm, colorbar=colorbar, labels=labels, **plotopts)        
             ax = pyplot.gcf().add_subplot(122)
-            BINoculars.plot.plot(space, pyplot.gcf(), ax, log=log, clipping=clipping, fit=fit, norm=norm, colorbar=colorbar, labels=labels, **plotopts)        
+            return BINoculars.plot.plot(space, pyplot.gcf(), ax, log=log, clipping=clipping, fit=fit, norm=norm, colorbar=colorbar, labels=labels, **plotopts)        
         else:
-            BINoculars.plot.plot(space, pyplot.gcf(), pyplot.gca(), log=log, clipping=clipping, fit=fit, norm=norm, colorbar=colorbar, labels=labels, **plotopts)
+            return BINoculars.plot.plot(space, pyplot.gcf(), pyplot.gca(), log=log, clipping=clipping, fit=fit, norm=norm, colorbar=colorbar, labels=labels, **plotopts)
     else:
         raise TypeError("'{0!r}' is not a BINoculars space".format(space))
 
diff --git a/BINoculars/backend.py b/BINoculars/backend.py
index 2b5e0cf..b29e12c 100644
--- a/BINoculars/backend.py
+++ b/BINoculars/backend.py
@@ -5,11 +5,15 @@ class ProjectionBase(util.ConfigurableObject):
     def parse_config(self, config):
         super(ProjectionBase, self).parse_config(config)
         res = config.pop('resolution')# or just give 1 number for all dimensions
+        self.config.limits = util.parse_pairs(config.pop('limits', None))#Optional, set the limits of the space object in projected coordinates. Syntax is same as numpy e.g. '0.3:-0.6, -1:5, :'
         labels = self.get_axis_labels()
+        if not self.config.limits is None:
+            if len(self.config.limits) != len(labels):
+                raise errors.ConfigError('dimension mismatch between projection axes ({0}) and limits specification ({1}) in {2}'.format(labels, self.config.limits, self.__class__.__name__))
         if ',' in res:
             self.config.resolution = util.parse_tuple(res, type=float)
             if not len(labels) == len(self.config.resolution):
-                raise errors.ConfigError('dimension mismatch between projection axes ({0}) and resolution specification ({1}) in {2}', labels, self.config.resolution, self.__class__.__name__)
+                raise errors.ConfigError('dimension mismatch between projection axes ({0}) and resolution specification ({1}) in {2}'.format(labels, self.config.resolution, self.__class__.__name__))
         else:
             self.config.resolution = tuple([float(res)] * len(labels))
 
diff --git a/BINoculars/backends/id03.py b/BINoculars/backends/id03.py
index c2ec41e..abce894 100644
--- a/BINoculars/backends/id03.py
+++ b/BINoculars/backends/id03.py
@@ -231,44 +231,61 @@ class ID03Input(backend.InputBase):
         if not len(scans):
             sys.stderr.write('error: no scans selected, nothing to do\n')
         for scanno in scans:
-            scan = self.get_scan(scanno)
+            util.status('processing scan {0}...'.format(scanno))
             if self.config.wait_for_data:
-                target = self.target(scan)
-                if not target == -1 and not self.is_zap(scan):
-                    for s in util.chunk_slicer(target, self.config.target_weight):
-                        if self.wait_for_points(scanno, s.stop, timeout=180):
-                           stop = self.get_scan(scanno).lines()
-                           yield backend.Job(scan=scanno, firstpoint=s.start, lastpoint=stop-1, weight=s.stop-s.start)
-                           break
-                        else:
-                           yield backend.Job(scan=scanno, firstpoint=s.start, lastpoint=s.stop-1, weight=s.stop-s.start)
-                elif not target == -1:
-                    if not self.wait_for_points(scanno, target, timeout=180):
-                        for s in util.chunk_slicer(target, self.config.target_weight):
-                           yield backend.Job(scan=scanno, firstpoint=s.start, lastpoint=s.stop-1, weight=s.stop-s.start)
-                else:
-                    step = self.config.target_weight
-                    for start, stop in itertools.izip(itertools.count(0,step), itertools.count(step,step)):
-                        if self.wait_for_points(scanno, stop, timeout=180):
-                           stop = self.get_scan(scanno).lines()
-                           yield backend.Job(scan=scanno, firstpoint=start, lastpoint=stop-1, weight=stop-start)
-                           break
-                        else:
-                           yield backend.Job(scan=scanno, firstpoint=start, lastpoint=stop-1, weight=stop-start)
+                for job in self.get_delayed_jobs(scanno):
+                    yield job
             else:
-                try:
-                    pointcount = scan.lines()
-                except specfile.error: # no points
-                    continue
-                next(self.get_images(scan, 0, pointcount-1, dry_run=True))# dryrun
+                scan = self.get_scan(scanno)
                 if self.config.pr:
-                    pointcount = self.config.pr[1] - self.config.pr[0]
-                    yield backend.Job(scan=scanno, firstpoint=self.config.pr[0], lastpoint=self.config.pr[1], weight=pointcount)
-                elif self.config.target_weight and pointcount > self.config.target_weight * 1.4:
+                    pointcount = self.config.pr[1] - self.config.pr[0] + 1
+                    start = self.config.pr[0]
+                else:
+                    start = 0
+                    try:
+                        pointcount = scan.lines()
+                    except specfile.error: # no points
+                        continue
+                next(self.get_images(scan, 0, pointcount-1, dry_run=True))# dryrun
+                if pointcount > self.config.target_weight * 1.4:
                     for s in util.chunk_slicer(pointcount, self.config.target_weight):
-                        yield backend.Job(scan=scanno, firstpoint=s.start, lastpoint=s.stop-1, weight=s.stop-s.start)
+                        yield backend.Job(scan=scanno, firstpoint=start+s.start, lastpoint=start+s.stop-1, weight=s.stop-s.start)
+                else:
+                    yield backend.Job(scan=scanno, firstpoint=start, lastpoint=start+pointcount-1, weight=pointcount)
+
+    def get_delayed_jobs(self, scanno):
+        scan = self.get_delayed_scan(scanno)
+
+        if self.config.pr:
+            firstpoint, lastpoint = self.config.pr#firstpoint is the first index to be included, lastpoint the last index to be included.
+        else:
+            firstpoint, lastpoint = 0, self.target(scan) - 1
+
+        pointcount = lastpoint - firstpoint + 1
+
+        if self.is_zap(scan):   #wait until the scan is finished.
+            if not self.wait_for_points(scanno, self.target(scan), timeout=self.config.timeout):    #wait for last datapoint
+                for s in util.chunk_slicer(pointcount, self.config.target_weight):
+                   yield backend.Job(scan=scanno, firstpoint=firstpoint+s.start, lastpoint=firstpoint+s.stop-1, weight=s.stop-s.start)
+            else:
+                raise errors.BackendError('Image collection timed out. Zapscan was probably aborted')
+        elif lastpoint >= 0:  #scanlength is known
+            for s in util.chunk_slicer(pointcount, self.config.target_weight):
+                if self.wait_for_points(scanno, firstpoint + s.stop, timeout=self.config.timeout):
+                   stop = self.get_scan(scanno).lines()
+                   yield backend.Job(scan=scanno, firstpoint=firstpoint+s.start, lastpoint=stop-1, weight=s.stop-s.start)
+                   break
+                else:
+                   yield backend.Job(scan=scanno, firstpoint=firstpoint+s.start, lastpoint=firstpoint+s.stop-1, weight=s.stop-s.start)
+        else:    #scanlength is unknown
+            step = int(self.config.target_weight / 1.4)
+            for start, stop in itertools.izip(itertools.count(0,step), itertools.count(step,step)):
+                if self.wait_for_points(scanno, stop, timeout=self.config.timeout):
+                   stop = self.get_scan(scanno).lines()
+                   yield backend.Job(scan=scanno, firstpoint=start, lastpoint=stop-1, weight=stop-start)
+                   break
                 else:
-                    yield backend.Job(scan=scanno, firstpoint=0, lastpoint=pointcount-1, weight=pointcount)
+                   yield backend.Job(scan=scanno, firstpoint=start, lastpoint=stop-1, weight=stop-start)
 
     def process_job(self, job):
         super(ID03Input, self).process_job(job)
@@ -305,7 +322,8 @@ class ID03Input(backend.InputBase):
             self.config.pr = util.parse_tuple(self.config.pr, length=2, type=int)
         self.config.sdd = float(config.pop('sdd'))# sample to detector distance (mm)
         self.config.pixelsize = util.parse_tuple(config.pop('pixelsize'), length=2, type=float)# pixel size x/y (mm) (same dimension as sdd)
-        self.config.wait_for_data = util.parse_bool(config.pop('wait_for_data', 'false'))
+        self.config.wait_for_data = util.parse_bool(config.pop('wait_for_data', 'false'))#Optional, if true wait until the data appears
+        self.config.timeout = int(config.pop('timeout', 180))#Optional, how long the script wait until it assumes the scan is not continuing
 
     def get_destination_options(self, command):
         if not command:
@@ -315,42 +333,58 @@ class ID03Input(backend.InputBase):
         return dict(first=min(scans), last=max(scans), range=','.join(str(scan) for scan in scans))
 
     # CONVENIENCE FUNCTIONS
-    _spec = None
     def get_scan(self, scannumber):
-        if self._spec is None:
-            self._spec = specfilewrapper.Specfile(self.config.specfile)
-        if self.config.wait_for_data:
-            util.status('waiting for scan {0}...'.format(scannumber))
-            delay = util.loop_delayer(5)
-            while 1:
-                try:
-                    return specfilewrapper.Specfile(self.config.specfile).select('{0}.1'.format(scannumber))
-                except specfile.error:
-                    next(delay)
-        else:
-            return self._spec.select('{0}.1'.format(scannumber))
+        spec = specfilewrapper.Specfile(self.config.specfile)
+        return spec.select('{0}.1'.format(scannumber))
 
-    def wait_for_points(self, scanno, stop, timeout=None):
+    def get_delayed_scan(self, scannumber, timeout = None):
         delay = util.loop_delayer(5)
         start = time.time()
-        while self.get_scan(scanno).lines() < stop:
-            util.status('waiting for scan {0}, point {1}...'.format(scanno, stop))
-            next(delay)
-            if timeout is not None and time.time() - start > timeout:
-                util.statusnl('scan {0} aborted at point {1}'.format(scanno, self.get_scan(scanno).lines()))
-                return True
-        return False
+        while 1:
+            try:
+                return self.get_scan(scannumber) #reload entire specfile
+            except specfile.error:
+                if timeout is not None and time.time() - start > timeout:
+                    raise errors.BackendError('Scan timed out. There is no data to process')
+                else:
+                    util.status('waiting for scan {0}...'.format(scannumber))
+                    next(delay)
 
-    @staticmethod
-    def target(scan):
-        if scan.command().startswith('ascan') or scan.command().startswith('hklscan') or scan.command().startswith('a2scan'):
+    def wait_for_points(self, scannumber, stop, timeout=None):
+        delay = util.loop_delayer(1)
+        start = time.time()
+
+        while 1:
+            scan = self.get_scan(scannumber)
+            try:
+                if scan.lines() >= stop:
+                    next(delay)    #time delay between specfile and edf file
+                    return False
+            except specfile.error:
+                pass
+            finally:
+                next(delay)
+
+            util.status('waiting for scan {0}, point {1}...'.format(scannumber, stop))
+            if (timeout is not None and time.time() - start > timeout) or self.is_aborted(scan):
+                try:
+                    util.statusnl('scan {0} aborted at point {1}'.format(scannumber, scan.lines()))
+                    return True
+                except specfile.error:
+                    raise errors.BackendError('Scan was aborted before images were collected. There is no data to process')
+
+
+    def target(self, scan):
+        if any(tuple(scan.command().startswith(pattern) for pattern in ['hklscan', 'a2scan', 'ascan', 'ringscan'])):
             return int(scan.command().split()[-2]) + 1
         elif scan.command().startswith('mesh'):
             return int(scan.command().split()[-6]) * int(scan.command().split()[-2]) + 1
+        elif scan.command().startswith('loopscan'):
+            return int(scan.command().split()[-3])
         elif scan.command().startswith('xascan'):
             params = numpy.array(scan.command().split()[-6:]).astype(float)
             return int(params[2] + 1 + (params[4] - 1) / params[5] * params[2])
-        elif is_zap(scan):
+        elif self.is_zap(scan):
             return int(scan.command().split()[-2])
         else:
             return -1
@@ -359,6 +393,13 @@ class ID03Input(backend.InputBase):
     def is_zap(scan):
         return scan.command().startswith('zap')
 
+    @staticmethod
+    def is_aborted(scan):
+        for line in scan.header('C'):
+            if 'Scan aborted' in line:
+                return True
+        return False
+
     def find_edfs(self, pattern, scanno):
         files = glob.glob(pattern)
         ret = {}
@@ -496,13 +537,13 @@ class EH1(ID03Input):
       
     def process_image(self, scanparams, pointparams, image):
         gamma, delta, theta, chi, phi, mu, mon, transm, hrx, hry = pointparams
+        wavelength, UB = scanparams
 
         if self.config.hr:
             zerohrx, zerohry = self.config.hr
             chi = (hrx - zerohrx) / numpy.pi * 180. / 1000
             phi = (hry - zerohry) / numpy.pi * 180. / 1000
 
-        wavelength, UB = scanparams
         if self.config.background:
             data = image / mon
         else:
@@ -705,7 +746,6 @@ class GisaxsDetector(ID03Input):
         ccdy, ccdz, theta, chi, phi, mu, mon, transm= pointparams
 
         image = numpy.rot90(image, self.config.drotation)
-        image = numpy.fliplr(image)
 
         wavelength, UB = scanparams
 
@@ -723,11 +763,16 @@ class GisaxsDetector(ID03Input):
         pixelsize = numpy.array(self.config.pixelsize)
         sdd = self.config.sdd 
 
-        app = numpy.arctan(pixelsize / sdd) * 180 / numpy.pi
-
         directbeam = (self.config.directbeam[0] - (ccdy - self.config.directbeam_coords[0]) * pixelsize[0], self.config.directbeam[1] - (ccdz - self.config.directbeam_coords[1]) * pixelsize[1])
-        gamma_range= app[0] * (numpy.arange(data.shape[0]) - directbeam[0]) - mu
-        delta_range= app[1] * (numpy.arange(data.shape[1]) - directbeam[1])
+        gamma_distance  = - pixelsize[1] * (numpy.arange(data.shape[1]) - directbeam[1])
+        delta_distance  = - pixelsize[0] * (numpy.arange(data.shape[0]) - directbeam[0])
+
+        gamma_range = numpy.arctan2(gamma_distance, sdd) / numpy.pi * 180 - mu
+        delta_range = numpy.arctan2(delta_distance, sdd) / numpy.pi * 180
+
+        #sample pixel distance
+        spd = numpy.sqrt(gamma_distance**2 + delta_distance**2 + sdd**2)
+        data *= spd**2 / sdd
 
         # masking
         gamma_range = gamma_range[self.config.ymask]
@@ -736,6 +781,7 @@ class GisaxsDetector(ID03Input):
 
         return intensity, (wavelength, UB, gamma_range, delta_range, theta, mu, chi, phi)
 
+
     def parse_config(self, config):
         super(GisaxsDetector, self).parse_config(config)
         self.config.drotation = int(config.pop('drotation', 0)) #Optional; Rotation of the detector, takes standard orientation by default. input 1 for 90 dgree rotation, 2 for 180 and 3 for 270.
@@ -763,6 +809,19 @@ class GisaxsDetector(ID03Input):
         params[:, MU] = scan.datacol('mucnt')[sl]
         return params
 
+    def find_edfs(self, pattern, scanno):
+        files = glob.glob(pattern)
+        ret = {}
+        for file in files:
+            try:
+                filename = os.path.basename(file).split('.')[0]
+                scan, point = filename.split('_')[-2:]
+                scan, point = int(scan), int(point)
+                if scan == scanno and point not in ret.keys():
+                    ret[point] = file
+            except ValueError:
+                continue
+        return ret
 
 def load_matrix(filename):
     if filename == None:
@@ -780,3 +839,4 @@ def load_matrix(filename):
     else:
        raise IOError('filename: {0} does not exist. Can not load matrix'.format(filename))
 
+
diff --git a/BINoculars/dispatcher.py b/BINoculars/dispatcher.py
index 5cdbc84..3b6a409 100755
--- a/BINoculars/dispatcher.py
+++ b/BINoculars/dispatcher.py
@@ -67,15 +67,15 @@ class DispatcherBase(util.ConfigurableObject):
         self.config.send_to_gui = util.parse_bool(config.pop('send_to_gui', 'false'))#previewing the data, if true, also specify host and port
 
     def send(self, spaces):#provides the possiblity to send the results to the gui over the network
-        if not self.config.send_to_gui or (self.config.host == None or self.config.host == None):#only continue of ip is specified and send_to_server is flagged
-            for space in spaces:
-                yield space
+        if self.config.send_to_gui or (self.config.host is not None and self.config.host is not None):#only continue of ip is specified and send_to_server is flagged
+             for sp in spaces:
+                if isinstance(sp, space.Space):
+                    util.socket_send(self.config.host, int(self.config.port), util.serialize(sp, ','.join(self.main.config.command)))
+                yield sp
         else:
-            for space in spaces:
-                util.socket_send(self.config.host, int(self.config.port), util.serialize(space, ','.join(self.main.config.command)))
-                yield space
+            for sp in spaces:
+                yield sp
          
-
     def has_specific_task(self):
         return False
 
@@ -171,26 +171,34 @@ class Oar(ReentrantBase):
     def process_jobs(self, jobs):
         self.configfiles = []
         self.intermediates = []
-        clusters = list(util.cluster_jobs(jobs, self.main.input.config.target_weight))
-        for i, jobscluster in enumerate(clusters, start=1):
+        clusters = util.cluster_jobs2(jobs, self.main.input.config.target_weight)
+        for jobscluster in clusters:
             uniq = util.uniqid()
             jobconfig = os.path.join(self.config.tmpdir, 'binoculars-{0}-jobcfg.zpi'.format(uniq))
             self.configfiles.append(jobconfig)
 
             config = self.main.clone_config()
-            if i == len(clusters):
-                config.dispatcher.sum = self.intermediates
-            else:
-                interm = os.path.join(self.config.tmpdir, 'binoculars-{0}-jobout.hdf5'.format(uniq))
-                self.intermediates.append(interm)
-                config.dispatcher.destination.set_tmp_filename(interm)
-                config.dispatcher.sum = ()
+            interm = os.path.join(self.config.tmpdir, 'binoculars-{0}-jobout.hdf5'.format(uniq))
+            self.intermediates.append(interm)
+            config.dispatcher.destination.set_tmp_filename(interm)
+            config.dispatcher.sum = ()
 
             config.dispatcher.action = 'process'
             config.dispatcher.jobs = jobscluster
             util.zpi_save(config, jobconfig)
             yield self.oarsub(jobconfig)
 
+        #if all jobs are sent to the cluster send the process that sums all other jobs
+        uniq = util.uniqid()
+        jobconfig = os.path.join(self.config.tmpdir, 'binoculars-{0}-jobcfg.zpi'.format(uniq))
+        self.configfiles.append(jobconfig)
+        config = self.main.clone_config()
+        config.dispatcher.sum = self.intermediates
+        config.dispatcher.action = 'process'
+        config.dispatcher.jobs = ()
+        util.zpi_save(config, jobconfig)
+        yield self.oarsub(jobconfig)
+
     def sum(self, results):
         jobs = list(results)
         jobscopy = jobs[:]
@@ -206,23 +214,10 @@ class Oar(ReentrantBase):
         if self.config.jobs:
             jobs = space.sum(self.send(self.main.process_job(job) for job in self.config.jobs))
         if self.config.sum:
-            sum = space.chunked_sum(space.Space.fromfile(src) for src in self.yield_when_exists(self.config.sum))
+            sum = space.chunked_sum(space.Space.fromfile(src) for src in util.yield_when_exists(self.config.sum))
         self.config.destination.store(jobs + sum)
 
-    ### UTILITY
-    @staticmethod
-    def yield_when_exists(files):
-        files = set(files)
-        polltime = 0
-        while files:
-            if time.time() - polltime < 5:
-                time.sleep(time.time() - polltime)
-            polltime = time.time()
-            exists = set(f for f in files if os.path.exists(f))
-            for e in exists:
-                yield e
-            files -= exists
-
+    ### calling OAR
     @staticmethod
     def subprocess_run(*command):
         process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -230,7 +225,6 @@ class Oar(ReentrantBase):
         retcode = process.poll()
         return retcode, output
 
-    ### calling OAR
     def oarsub(self, *args):
         command = '{0} process {1}'.format(self.config.executable, ' '.join(args))
         ret, output = self.subprocess_run('oarsub', '-l {0}'.format(self.config.oarsub_options), command)
@@ -239,6 +233,7 @@ class Oar(ReentrantBase):
             for line in lines:
                 if line.startswith('OAR_JOB_ID='):
                     void, jobid = line.split('=')
+                    util.status('{0}: Launched job {1}'.format(time.ctime(), jobid))
                     return jobid.strip()
         return False
 
diff --git a/BINoculars/errors.py b/BINoculars/errors.py
index fe5825b..dc2d8ac 100644
--- a/BINoculars/errors.py
+++ b/BINoculars/errors.py
@@ -18,6 +18,8 @@ class SubprocessError(ExceptionBase):
 class BackendError(ExceptionBase):
     pass
 
+class CommunicationError(ExceptionBase):
+    pass
 
 def addmessage(args, errormsg):
     if not args:
diff --git a/BINoculars/main.py b/BINoculars/main.py
index 8dc42d0..963dcca 100755
--- a/BINoculars/main.py
+++ b/BINoculars/main.py
@@ -88,9 +88,10 @@ class Main(object):
             labels = self.projection.get_axis_labels()
             for intensity, params in self.input.process_job(job):
                 coords = self.projection.project(*params)
-                yield space.Space.from_image(res, labels, coords, intensity)
+                yield space.Space.from_image(res, labels, coords, intensity, limits = self.projection.config.limits)
         jobspace = space.chunked_sum(generator(), chunksize=25)
-        jobspace.metadata.add_dataset(self.input.metadata)
+        if isinstance(jobspace, space.Space):
+            jobspace.metadata.add_dataset(self.input.metadata)
         return jobspace
 
     def clone_config(self):
@@ -123,7 +124,8 @@ class Split(Main): #completely ignores the dispatcher, just yields a space per i
         labels = self.projection.get_axis_labels()
         for intensity, params in self.input.process_job(job):
             coords = self.projection.project(*params)
-            yield space.Space.from_image(res, labels, coords, intensity)
+            yield space.Space.from_image(res, labels, coords, intensity, limits = self.projection.config.limits)
+
 
     def run(self):
         for job in self.input.generate_jobs(self.command):
diff --git a/BINoculars/plot.py b/BINoculars/plot.py
index 713972c..a25a21c 100644
--- a/BINoculars/plot.py
+++ b/BINoculars/plot.py
@@ -160,19 +160,28 @@ def plot(space, fig, ax, log=True, loglog = False, clipping=0.0, fit=None, norm=
             raise ValueError("For 3D plots, the 'ax' parameter must be an Axes3D instance (use for example gca(projection='3d') to get one)")

 

         cmap = getattr(matplotlib.cm, plotopts.pop('cmap', 'jet'))

-        norm = get_clipped_norm(space.get_masked(), clipping, log)

+        if not norm is None:

+            norm = get_clipped_norm(space.get_masked(), clipping, log)

 

-        gridx, gridy, gridz = space.get_grid()

-        p1 = ax.plot_surface(gridx[0,:,:], gridy[0,:,:], gridz[0,:,:],  facecolors=cmap(norm(space.project(0).get_masked())), shade=False, cstride=1, rstride=1)

-        p2 = ax.plot_surface(gridx[:,-1,:], gridy[:,-1,:], gridz[:,-1,:], facecolors=cmap(norm(space.project(1).get_masked())), shade=False, cstride=1, rstride=1)

-        p3 = ax.plot_surface(gridx[:,:,0], gridy[:,:,0], gridz[:,:,0],  facecolors=cmap(norm(space.project(2).get_masked())), shade=False, cstride=1, rstride=1)

+        data = space.get()

+        mask = numpy.bitwise_or(~numpy.isfinite(data), data == 0)

+        gridx, gridy, gridz = tuple(grid[~mask] for grid in space.get_grid())

+        

+        im = ax.scatter(gridx, gridy, gridz,  c=cmap(norm(data[~mask])), marker = ',', alpha = 0.7,linewidths = 0)

+

+        #p1 = ax.plot_surface(gridx[0,:,:], gridy[0,:,:], gridz[0,:,:],  facecolors=cmap(norm(space.project(0).get_masked())), shade=False, cstride=1, rstride=1)

+        #p2 = ax.plot_surface(gridx[:,-1,:], gridy[:,-1,:], gridz[:,-1,:], facecolors=cmap(norm(space.project(1).get_masked())), shade=False, cstride=1, rstride=1)

+        #p3 = ax.plot_surface(gridx[:,:,0], gridy[:,:,0], gridz[:,:,0],  facecolors=cmap(norm(space.project(2).get_masked())), shade=False, cstride=1, rstride=1)

 

         if labels:

             ax.set_xlabel(space.axes[0].label)

             ax.set_ylabel(space.axes[1].label)

             ax.set_zlabel(space.axes[2].label)

         

-        return p1 + p2 + p3

+        if fig._draggablecbar:

+            fig._draggablecbar.disconnect()

+

+        return im

 

     elif space.dimension > 3:

         raise ValueError("Cannot plot 4 or higher dimensional spaces, use projections or slices to decrease dimensionality.")

diff --git a/BINoculars/space.py b/BINoculars/space.py
index 6e1ad48..eb87dfb 100644..100755
--- a/BINoculars/space.py
+++ b/BINoculars/space.py
@@ -65,12 +65,16 @@ class Axis(object):
                 raise IndexError('stride not supported')
             if key.start is None:
                 start = 0
+            elif key.start < 0:
+                raise IndexError('key out of range')
             elif isinstance(key.start, int):
                 start = key.start
             else:
                 raise IndexError('key start must be integer')
             if key.stop is None:
                 stop = len(self)
+            elif key.stop > len(self):
+                raise IndexError('key out of range')
             elif isinstance(key.stop, int):
                 stop = key.stop
             else:
@@ -164,6 +168,8 @@ class Axis(object):
                start = self.restrict(value.start)
            if value.stop is None:
                stop = None
+           if value.stop == self.max:
+               stop = None
            else:
                stop = self.restrict(value.stop)
            if start is not None and stop is not None and start > stop:
@@ -199,24 +205,22 @@ class Axes(object):
     def fromfile(cls, filename):
         with util.open_h5py(filename, 'r') as fp:
             try:
-                if 'axes' in fp:
-                    # old style, float min/max
+                if 'axes' in fp and 'axes_labels' in fp:
+                    # oldest style, float min/max
                     return cls(tuple(Axis(min, max, res, lbl) for ((min, max, res), lbl) in zip(fp['axes'], fp['axes_labels'])))
+                elif 'axes' in fp:#new
+                    return cls(tuple(Axis(int(imin), int(imax), res, lbl) for ((imin, imax, res), lbl) in zip(fp['axes'].values(), fp['axes'].keys())))
                 else:
-                    # new style, integer min/max
+                    # older style, integer min/max
                     return cls(tuple(Axis(imin, imax, res, lbl) for ((imin, imax), res, lbl) in zip(fp['axes_range'], fp['axes_res'], fp['axes_labels'])))
             except (KeyError, TypeError) as e:
                 raise errors.HDF5FileError('unable to load axes definition from HDF5 file {0}, is it a valid BINoculars file? (original error: {1!r})'.format(filename, e))
 
     def tofile(self, filename):
         with util.open_h5py(filename, 'w') as fp:
-            range = fp.create_dataset('axes_range', [len(self.axes), 2], dtype=int)
-            res = fp.create_dataset('axes_res', [len(self.axes)], dtype=float)
-            labels = fp.create_dataset('axes_labels', [len(self.axes)], dtype=h5py.special_dtype(vlen=str))
-            for i, ax in enumerate(self.axes):
-                range[i, :] = ax.imin, ax.imax
-                res[i] = ax.res
-                labels[i] = ax.label
+            axes = fp.create_group('axes')
+            for ax in self.axes:
+                axes.create_dataset(ax.label, data = [ax.imin, ax.imax, ax.res])
 
     def toarray(self):
         return numpy.vstack(numpy.hstack([str(ax.imin), str(ax.imax), str(ax.res), ax.label]) for ax in self.axes)
@@ -284,22 +288,34 @@ class Axes(object):
 class EmptySpace(object):
     """Convenience object for sum() and friends. Treated as zero for addition.
     Does not share a base class with Space for simplicity."""
-
+    def __init__(self,config=None, metadata=None):
+        self.config = config
+        self.metadata = metadata
+        
     def __add__(self, other):
-        if not isinstance(other, Space):
+        if not isinstance(other, Space) and not isinstance(other, EmptySpace):
             return NotImplemented
         return other
 
     def __radd__(self, other):
-        if not isinstance(other, Space):
+        if not isinstance(other, Space) and not isinstance(other, EmptySpace):
             return NotImplemented
         return other
 
     def __iadd__(self, other):
-        if not isinstance(other, Space):
+        if not isinstance(other, Space) and not isinstance(other, EmptySpace):
             return NotImplemented
         return other
 
+    def tofile(self, filename):
+        """Store EmptySpace in HDF5 file."""
+        with util.atomic_write(filename) as tmpname:
+            with util.open_h5py(tmpname, 'w') as fp:
+                fp.attrs['type'] = 'Empty'           
+
+    def __repr__(self):
+        return '{0.__class__.__name__}'.format(self)
+  
 
 class Space(object):
     """Main data-storing object in BINoculars. 
@@ -617,6 +633,7 @@ class Space(object):
         intensity = intensity[valid]
         if not intensity.size:
             return
+
         coordinates = tuple(coord[valid] for coord in coordinates)
 
         indices = numpy.array(tuple(ax.get_index(coord) for (ax, coord) in zip(self.axes, coordinates)))
@@ -631,7 +648,7 @@ class Space(object):
         self.contributions.ravel()[:contributions.size] += contributions
 
     @classmethod
-    def from_image(cls, resolutions, labels, coordinates, intensity):
+    def from_image(cls, resolutions, labels, coordinates, intensity, limits = None):
         """Create Space from image data. 
 
         resolutions   n-tuple of axis resolutions
@@ -639,6 +656,21 @@ class Space(object):
         coordinates   n-tuple of data coordinate arrays
         intensity     data intensity array"""
 
+        if limits is not None:
+            invalid = numpy.zeros(intensity.shape).astype(numpy.bool)
+            for coord, sl in zip(coordinates, limits):
+                if sl.start == None and sl.stop != None:
+                    invalid += coord > sl.stop
+                elif sl.start != None and sl.stop == None:
+                    invalid += coord < sl.start
+                elif sl.start != None and sl.stop != None:
+                    invalid += numpy.bitwise_or(coord < sl.start, coord > sl.stop)
+
+            if numpy.all(invalid == True):
+                return EmptySpace()
+            coordinates = tuple(coord[~invalid] for coord in coordinates)
+            intensity = intensity[~invalid]
+
         axes = tuple(Axis(coord.min(), coord.max(), res, label) for res, label, coord in zip(resolutions, labels, coordinates))
         newspace = cls(axes)
         newspace.process_image(coordinates, intensity)
@@ -648,6 +680,7 @@ class Space(object):
         """Store Space in HDF5 file."""
         with util.atomic_write(filename) as tmpname:
             with util.open_h5py(tmpname, 'w') as fp:
+                fp.attrs['type'] = 'Space'
                 self.config.tofile(fp)
                 self.axes.tofile(fp)
                 self.metadata.tofile(fp)
@@ -658,10 +691,14 @@ class Space(object):
     def fromfile(cls, file, key=None):
         """Load Space from HDF5 file.
 
-        file      filename string or h5py.File instance
+        file      filename string or h5py.Group instance
         key       sliced (subset) loading, should be an n-tuple of slice()s in data coordinates"""
         try:
             with util.open_h5py(file, 'r') as fp:
+                if 'type' in fp.attrs.keys():
+                    if fp.attrs['type'] == 'Empty':
+                        return EmptySpace()
+                
                 axes = Axes.fromfile(fp)
                 config = util.ConfigFile.fromfile(fp)
                 metadata = util.MetaData.fromfile(fp)
@@ -678,6 +715,7 @@ class Space(object):
                     fp['contributions'].read_direct(space.contributions, key)
                 except (KeyError, TypeError) as e:
                     raise errors.HDF5FileError('unable to load Space from HDF5 file {0}, is it a valid BINoculars file? (original error: {1!r})'.format(file, e))
+
         except IOError as e:
             raise errors.HDF5FileError("unable to open '{0}' as HDF5 file (original error: {1!r})".format(file, e))
         return space
@@ -711,7 +749,9 @@ def union_unequal_axes(axes):
 
 def sum(spaces):
     """Calculate sum of iterable of Space instances."""
-    spaces = tuple(spaces)
+    spaces = tuple(space for space in spaces if not isinstance(space, EmptySpace))
+    if len(spaces) == 0:
+        return EmptySpace()
     if len(spaces) == 1:
         return spaces[0]
     if len(set(space.dimension for space in spaces)) != 1:
diff --git a/BINoculars/util.py b/BINoculars/util.py
index d0ef403..007c2b6 100755
--- a/BINoculars/util.py
+++ b/BINoculars/util.py
@@ -285,7 +285,7 @@ def parse_multi_range(s):
     ranges = s.split(',')
     for r in ranges:
         out.extend(parse_range(r))
-    return numpy.asarray(out)
+    return out
 
 def parse_tuple(s, length=None, type=str):
     t = tuple(type(i) for i in s.split(','))
@@ -301,6 +301,25 @@ def parse_bool(s):
         return False
     raise ValueError("invalid input for boolean: '{0}'".format(s))
 
+def parse_pairs(s):
+    if not s:
+        return s
+    pairs = s.split(',')
+    limits = []
+    for pair in pairs:
+        mi, ma = tuple(m.strip() for m in pair.split(':'))
+        if mi == '' and ma == '':
+            limits.append(slice(None))
+        elif mi == '':
+            limits.append(slice(None, float(ma)))
+        elif ma == '':
+            limits.append(slice(float(mi), None))
+        else:
+            if float(ma) < float(mi):
+                raise ValueError("invalid input. maximum is larger than minimum: '{0}'".format(s))
+            else:
+                limits.append(slice(float(mi), float(ma)))
+    return limits
 
 class MetaBase(object):
     def __init__(self, label = None, section = None):
@@ -458,12 +477,15 @@ class ConfigFile(MetaBase):
         with open_h5py(filename, 'r') as fp:
             try:
                 config = fp['configuration']
-                if 'command' in config.attrs.keys():
+                if 'command' in config.attrs:
                     configobj.command = json.loads(config.attrs['command'])
+                for section in config:
+                    if isinstance(config[section],  h5py._hl.group.Group):#new
+                        setattr(configobj, section, dict((key, config[section][key].value) for key in config[section]))
+                    else:#old
+                        setattr(configobj, section, dict(config[section]))
             except KeyError as e:
-                config = [] # when config is not present, proceed without Error
-            for section in config:
-                setattr(configobj, section, dict(config[section]))
+                pass # when config is not present, proceed without Error
         return configobj
 
     @classmethod
@@ -484,17 +506,14 @@ class ConfigFile(MetaBase):
 
     def tofile(self, filename):
         with open_h5py(filename, 'w') as fp:
-            dt = h5py.special_dtype(vlen=str)
             conf = fp.create_group('configuration')
             conf.attrs['origin'] = str(self.origin)
             conf.attrs['command'] = json.dumps(self.command)
             for section in self.sections:
+                sectiongroup = conf.create_group(section)
                 s = getattr(self, section)
-                if len(s):
-                    dataset = conf.create_dataset(section, (len(s),2), dtype=dt)
-                    for i,entry in zip(range(len(s)), s):
-                        dataset[i, 0] = entry
-                        dataset[i, 1] = s[entry]
+                for key in s.keys():
+                    sectiongroup.create_dataset(key, data = s[key])
 
     def totxtfile(self, filename):
         with open(filename, 'w') as fp:
@@ -633,12 +652,19 @@ def space_to_txt(space, filename):
 
 @contextlib.contextmanager
 def open_h5py(file, mode):
-    if isinstance(file, h5py.File):
+    if isinstance(file, h5py._hl.group.Group):
         yield file
     else:
         with h5py.File(file, mode) as fp:
-            yield fp
-
+            if mode == 'w':
+                if not 'binoculars' in fp:
+                    fp.create_group('binoculars')
+                yield fp['binoculars']
+            if mode == 'r':
+                if 'binoculars' in fp:
+                    yield fp['binoculars']
+                else:
+                    yield fp
 
 ### VARIOUS
 
@@ -683,6 +709,19 @@ def cluster_jobs(jobs, target_weight):
                 cluster.append(jobs.pop(i))
         yield cluster
 
+def cluster_jobs2(jobs, target_weight):
+    """Taking the first n jobs that together add up to target_weight.
+       Here as opposed to cluster_jobs the total number of jobs does not have to be known beforehand
+    """
+    jobslist = []
+    for job in jobs:
+       jobslist.append(job)
+       if sum(j.weight for j in jobslist) >= target_weight:
+           yield jobslist[:]
+           jobslist = []
+    if len(jobslist) > 0:#yield the remainder of the jobs
+        yield jobslist[:]
+
 def loop_delayer(delay):
     """Delay a loop such that it runs at most once every 'delay' seconds. Usage example:
     delay = loop_delayer(5)
@@ -812,12 +851,14 @@ def serialize(space, command):
     message.seek(0)
     message.write(struct.pack('QQQQQQ', commandlength, configlength, metalength, arraylength, photonlength, contributionlength))
     message.seek(0)
+
     return message
 
 def packet_slicer(length, size = 1024):#limit the communication to 1024 bytes
-    packets = [size] * (length / size)
-    packets.append(length % size)
-    return packets
+    while length > size:
+        length -= size
+        yield size
+    yield length
 
 def socket_send(ip, port, mssg):
     try:
@@ -840,10 +881,13 @@ def socket_recieve(RequestHandler):#pass one the handler to deal with incoming d
     def get_msg(length):
         msg = StringIO.StringIO()
         for packet in packet_slicer(length):
-            msg.write(RequestHandler.request.recv(packet))
+            p = RequestHandler.request.recv(packet, socket.MSG_WAITALL) #wait for full mssg
+            msg.write(p)
+        if msg.len != length:
+            raise  errors.CommunicationError('recieved message is too short. expected length {0}, recieved length {1}'.format(length, msg.len))
         msg.seek(0)
         return msg
 
-    command, config, metadata, axes, photons, contributions = tuple(get_msg(msglength) for msglength in struct.unpack('QQQQQQ',RequestHandler.request.recv(48)))
+    command, config, metadata, axes, photons, contributions = tuple(get_msg(msglength) for msglength in struct.unpack('QQQQQQ',RequestHandler.request.recv(48, socket.MSG_WAITALL)))
     return command.read(), config.read(), metadata.read(), numpy.load(axes), numpy.load(photons), numpy.load(contributions)
 
diff --git a/binoculars.py b/binoculars.py
index edf5b5f..58508dd 100755
--- a/binoculars.py
+++ b/binoculars.py
@@ -303,7 +303,7 @@ def command_process(args):
     import BINoculars.main
 
     BINoculars.util.register_python_executable(__file__)
-    BINoculars.main.Main.from_args(args)
+    BINoculars.main.Main.from_args(args)# start of main thread
 
 
 
diff --git a/gui.py b/gui.py
index 80c30e9..2025a52 100755
--- a/gui.py
+++ b/gui.py
@@ -5,6 +5,8 @@ from PyQt4 import QtGui, QtCore, Qt
 import BINoculars.main, BINoculars.space, BINoculars.plot,  BINoculars.util
 import numpy
 import json
+import itertools
+from mpl_toolkits.mplot3d import Axes3D
 
 import signal
 import subprocess
@@ -234,15 +236,19 @@ class Window(QtGui.QMainWindow):
         edit.addAction(merge) 
         edit.addAction(subtract) 
 
-        start_server = QtGui.QAction("Start server", self)  
-        start_server.triggered.connect(self.run_server)
+        start_server = QtGui.QAction("Start server queue", self)  
+        start_server.triggered.connect(lambda: self.open_server(startq = True))
 
-        stop_server = QtGui.QAction("Stop server", self)  
+        stop_server = QtGui.QAction("Stop server queue", self)  
         stop_server.triggered.connect(self.kill_server)
 
+        recieve = QtGui.QAction("Open for spaces", self)  
+        recieve.triggered.connect(lambda: self.open_server(startq = False))
+
         serve = menu_bar.addMenu("&Serve") 
         serve.addAction(start_server)
         serve.addAction(stop_server)
+        serve.addAction(recieve)
 
         self.tab_widget = QtGui.QTabWidget(self)
         self.tab_widget.setTabsClosable(True)
@@ -328,7 +334,7 @@ class Window(QtGui.QMainWindow):
                 else:
                     widget.addspace(str(name), False)
             except Exception as e:
-                QtGui.QMessageBox.critical(self, 'Import spaces', 'Unable to import space {}: {}'.format(fname, e))
+                QtGui.QMessageBox.critical(self, 'Import spaces', 'Unable to import space {}: {}'.format(str(name), e))
 
     def exportspace(self):
         widget = self.tab_widget.currentWidget()
@@ -381,7 +387,7 @@ class Window(QtGui.QMainWindow):
             except Exception as e:
                 QtGui.QMessageBox.critical(self, 'Import spaces', 'Unable to import space {}: {}'.format(fname, e))
 
-    def run_server(self):
+    def open_server(self, startq = True):
         if len(self.threads) != 0:
             print 'Server already running'
         else:
@@ -393,9 +399,9 @@ class Window(QtGui.QMainWindow):
 
             self.ip, self.port = server.server_address
 
-            cmd = ['python', os.path.join(os.path.dirname(__file__), 'server.py'), str(self.ip), str(self.port)]
-            self.pro = subprocess.Popen(cmd, stdin=None, stdout=None, stderr=None, preexec_fn=os.setsid) 
-
+            if startq:
+                cmd = ['python', os.path.join(os.path.dirname(__file__), 'server.py'), str(self.ip), str(self.port)]
+                self.pro = subprocess.Popen(cmd, stdin=None, stdout=None, stderr=None, preexec_fn=os.setsid) 
 
             server_thread = threading.Thread(target=server.serve_forever)
             server_thread.daemon = True
@@ -407,7 +413,8 @@ class Window(QtGui.QMainWindow):
             self.threads.append(updater)
             updater.start()
 
-            #print 'GUI server started running at ip {0} and port {1}.'.format(self.ip, self.port)
+            if not startq:
+                print 'GUI server started running at ip {0} and port {1}.'.format(self.ip, self.port)
 
     def kill_server(self):
         if len(self.threads) == 0:
@@ -434,19 +441,30 @@ class Window(QtGui.QMainWindow):
         index = names.index('server')
         serverwidget = self.tab_widget.widget(index)
 
-        while not self.q.empty():
-            command, space = self.q.get()
+        while not self.threads[0].fq.empty():
+            command, space = self.threads[0].fq.get()
             serverwidget.table.addfromserver(command, space)
             serverwidget.table.select()
             if serverwidget.auto_update.isChecked():
                 serverwidget.limitwidget.refresh()
 
 class UpdateThread(QtCore.QThread):
+    fq = Queue.Queue()
     data_found = QtCore.pyqtSignal(object)
     def run(self):
-        delay = BINoculars.util.loop_delayer(1)        
+        delay = BINoculars.util.loop_delayer(1)
+        jobs = []
+        labels = []        
         while 1:
             if not self.q.empty():
+                command, space = self.q.get()
+                if command in labels:
+                    jobs[labels.index(command)].append(space)
+                else:
+                    jobs.append([space])
+                    labels.append(command)
+            elif self.q.empty() and len(jobs) > 0:
+                self.fq.put((labels.pop(), BINoculars.space.sum(jobs.pop())))
                 self.data_found.emit('data found')
             else:
                 next(delay)
@@ -473,16 +491,23 @@ class HiddenToolbar(NavigationToolbar2QTAgg):
         self.update_sliders = update_sliders
         self.zoom()
 
+        self.threed = False
+
     def mouse_move(self, event):
-        self.show_coords(event)
+        if not self.threed:
+            self.show_coords(event)
 
     def press_zoom(self, event):
         super(HiddenToolbar, self).press_zoom(event)
-        self.plotaxes = event.inaxes
+        if not self.threed:
+            self.inaxes = event.inaxes
 
     def release_zoom(self, event):
         super(HiddenToolbar, self).release_zoom(event)
-        self.update_sliders(self.plotaxes)
+        if not self.threed:
+            self.update_sliders(self.inaxes)
+
+
 
 class ProjectWidget(QtGui.QWidget):
     def __init__(self, filelist, key = None, projection = None, parent = None):
@@ -510,7 +535,7 @@ class ProjectWidget(QtGui.QWidget):
         self.loggroup.addButton(self.log)
         self.loggroup.addButton(self.loglog)
 
-        self.swap_axes = QtGui.QCheckBox('swap ax', self)
+        self.swap_axes = QtGui.QCheckBox('ax', self)
         self.swap_axes.setChecked(False)
         QtCore.QObject.connect(self.swap_axes, QtCore.SIGNAL("stateChanged(int)"), self.plot)
 
@@ -522,6 +547,10 @@ class ProjectWidget(QtGui.QWidget):
         self.legend.setChecked(True)
         QtCore.QObject.connect(self.legend, QtCore.SIGNAL("stateChanged(int)"), self.plot)
 
+        self.threed = QtGui.QCheckBox('3d', self)
+        self.threed.setChecked(False)
+        QtCore.QObject.connect(self.threed, QtCore.SIGNAL("stateChanged(int)"), self.plot)
+
         self.auto_update = QtGui.QCheckBox('auto', self)
         self.auto_update.setChecked(True)
 
@@ -568,6 +597,7 @@ class ProjectWidget(QtGui.QWidget):
         self.group = QtGui.QButtonGroup(self)
         for label in ['stack', 'grid']:
             rb = QtGui.QRadioButton(label, self.control_widget)
+            rb.setChecked(True)
             self.group.addButton(rb)
             radiobox.addWidget(rb)
 
@@ -578,6 +608,7 @@ class ProjectWidget(QtGui.QWidget):
         datarangebox = QtGui.QHBoxLayout() 
         datarangebox.addWidget(self.samerange)
         datarangebox.addWidget(self.legend)
+        datarangebox.addWidget(self.threed)
         datarangebox.addWidget(self.swap_axes)
         datarangebox.addWidget(self.auto_update)
 
@@ -691,6 +722,8 @@ class ProjectWidget(QtGui.QWidget):
             return norm
 
     def plot(self):
+        if len(self.table.plotaxes) == 0:
+            return
         self.figure.clear()
         self.parent.statusbar.clearMessage()
 
@@ -709,12 +742,19 @@ class ProjectWidget(QtGui.QWidget):
 
         for i, filename in enumerate(self.table.selection):
             axes = self.table.getax(filename)
-            space = self.table.getspace(filename, axes.restricted_key(self.key))
+            rkey =  axes.restricted_key(self.key)
+            if rkey == None:
+                space = self.table.getspace(filename)
+            else:
+                space = self.table.getspace(filename, rkey)
             projection = [ax for ax in self.projection if ax in space.axes]
             if projection:
                 space = space.project(*projection)
-            if len(space.axes) > 2 or len(space.axes) == 0:
-                self.errormessage('choose suitable number of projections, plotting only in 1D and 2D')
+            dimension = space.dimension
+            if dimension == 0:
+                self.errormessage('Choose suitable number of projections')
+            if dimension == 3 and not self.threed.isChecked():
+                self.errormessage('Switch on 3D plotting, only works with small spaces')
             spaces.append(space)
 
         self.datamin = []
@@ -725,28 +765,37 @@ class ProjectWidget(QtGui.QWidget):
                 data = data[data > 0]
             self.datamin.append(data.min())
             self.datamax.append(data.max())
+
         norm = self.get_normlist()
 
+        if dimension == 1 or dimension == 2:
+            self.toolbar.threed = False
+        else:
+            self.toolbar.threed = True
+
         for i,space in enumerate(spaces):
             filename = self.table.selection[i]
             basename = os.path.splitext(os.path.basename(filename))[0]
             if plotcount > 1:
-                if space.dimension == 1 and (plotoption == 'stack' or plotoption == None):
+                if dimension == 1 and (plotoption == 'stack' or plotoption == None):
                     self.ax = self.figure.add_subplot(111)
-                if space.dimension == 2 and plotoption != 'grid':
+                if dimension == 2 and plotoption != 'grid':
                     sys.stderr.write('warning: stack display not supported for multi-file-plotting, falling back to grid\n')
                     plotoption = 'grid'
-                elif space.dimension > 3:
+                elif dimension > 3:
                     sys.stderr.write('error: cannot display 4 or higher dimensional data, use --project or --slice to decrease dimensionality\n')
                     sys.exit(1)
             else:
                  self.ax = self.figure.add_subplot(111)
 
             if plotoption == 'grid':
-                self.ax = self.figure.add_subplot(plotrows, plotcolumns, i+1)
+                if dimension == 1 or dimension == 2:
+                    self.ax = self.figure.add_subplot(plotrows, plotcolumns, i+1)
+                elif self.threed.isChecked():
+                    self.ax = self.figure.gca(projection='3d')
                 self.ax.set_title(basename)
 
-            if space.dimension == 2 and self.swap_axes.checkState():
+            if dimension == 2 and self.swap_axes.checkState():
                 space = space.reorder(list(ax.label for ax in space.axes)[::-1])
 
             self.ax.space = space
@@ -754,7 +803,7 @@ class ProjectWidget(QtGui.QWidget):
 
             self.figure_images.append(im)
         
-        if space.dimension == 1 and self.legend.checkState():
+        if dimension == 1 and self.legend.checkState():
             self.ax.legend()
         
         self.update_figure_range(self.key_to_str(self.key))
@@ -792,12 +841,13 @@ class ProjectWidget(QtGui.QWidget):
         if len(key) == 0:
             return
         for ax in self.figure.axes:
+            plotaxes = self.table.plotaxes
             xlabel, ylabel = ax.get_xlabel(), ax.get_ylabel()
-            if xlabel in self.table.plotaxes:
-                xindex = self.table.plotaxes.index(xlabel)
+            if xlabel in plotaxes:
+                xindex = plotaxes.index(xlabel)
                 ax.set_xlim(key[xindex][0], key[xindex][1])
-            if ylabel in self.table.plotaxes:
-                yindex = self.table.plotaxes.index(ylabel)
+            if ylabel in plotaxes:
+                yindex = plotaxes.index(ylabel)
                 ax.set_ylim(key[yindex][0], key[yindex][1])
         self.canvas.draw()
 
@@ -854,7 +904,7 @@ class ProjectWidget(QtGui.QWidget):
     def addspace(self,filename = None, add = False):
         if filename == None:
             filename = str(QtGui.QFileDialog.getOpenFileName(self, 'Open Project', '.', '*.hdf5'))
-        self.table.addspace(filename, add)
+        self.table.add_space(filename, add)
 
     def save(self):
         dialog = QtGui.QFileDialog(self, "Save image");
@@ -903,118 +953,127 @@ class ProjectWidget(QtGui.QWidget):
 def short_filename(filename):
     return filename.split('/')[-1].split('.')[0]
 
+class SpaceContainer(QtGui.QTableWidgetItem):
+    def __init__(self, label, space=None):
+        super(SpaceContainer, self).__init__(short_filename(label))
+        self.label = label
+        self.space = space
+
+    def get_space(self, key = None):
+        if self.space == None:
+            return BINoculars.space.Space.fromfile(self.label, key = key)
+        else:
+           if key == None:
+                key = Ellipsis
+           return self.space[key]
+
+    def get_ax(self):
+        if self.space == None:
+           return BINoculars.space.Axes.fromfile(self.label)
+        else:
+           return self.space.axes
+
+    def add_to_space(space):
+        if self.space == None:
+            newspace = BINoculars.space.Space.fromfile(self.label) + space
+            newspsace.tofile(self.label)
+        else:
+            self.space += space
+
 class TableWidget(QtGui.QWidget):
     def __init__(self, filelist = [],parent=None):
         super(TableWidget, self).__init__(parent)
 
         hbox = QtGui.QHBoxLayout()
-        self.plotaxes = []
 
-        self.table = QtGui.QTableWidget(0, 3)
-        self.table.setHorizontalHeaderLabels(['filename','labels', 'remove'])
+        self.table = QtGui.QTableWidget(0, 4)
+        self.table.setHorizontalHeaderLabels(['', 'filename','labels', 'remove'])
         
-        for index, width in enumerate([150,50,70]):
+        for index, width in enumerate([25,150,50,70]):
             self.table.setColumnWidth(index, width)
 
         for filename in filelist:
-            self.addspace(filename)
+            self.add_space(filename)
 
         hbox.addWidget(self.table)
         self.setLayout(hbox)
 
-    def addspace(self, filename, add = True, space = None):
-        def remove_callback(filename):
-            return lambda: self.remove(filename)
-
+    def add_space(self, filename, add = True, space = None):
         index = self.table.rowCount()
         self.table.insertRow(index)
 
-        if space == None:
-            axes = BINoculars.space.Axes.fromfile(filename)
-        else:
-            axes = space.axes
-
-        checkboxwidget = QtGui.QCheckBox(short_filename(filename))
-        checkboxwidget.setChecked(True)
-        checkboxwidget.filename = filename
-        checkboxwidget.space = space        
+        checkboxwidget = QtGui.QCheckBox()
+        checkboxwidget.setChecked(add)
         checkboxwidget.clicked.connect(self.select)
         self.table.setCellWidget(index,0, checkboxwidget)
 
-        item = QtGui.QTableWidgetItem(','.join(list(ax.label.lower() for ax in axes)))
-        self.table.setItem(index, 1, item)
+        container = SpaceContainer(filename, space)
+        self.table.setItem(index, 1, container)
+
+        item = QtGui.QTableWidgetItem(','.join(list(ax.label.lower() for ax in container.get_ax())))
+        self.table.setItem(index, 2, item)
 
         buttonwidget = QtGui.QPushButton('remove')
-        buttonwidget.clicked.connect(remove_callback(filename))
-        self.table.setCellWidget(index,2, buttonwidget)
+        buttonwidget.clicked.connect(lambda: self.remove(filename))
+        self.table.setCellWidget(index,3, buttonwidget)
         if add:
             self.select()
 
     def addfromserver(self, command, space):
         if not command in self.filelist:
-            self.addspace(command, add = False, space = space)
+            self.add_space(command, add = False, space = space)
         else:
-            checkbox = self.table.cellWidget(self.filelist.index(short_filename(command)), 0)
-            checkbox.space += space
+            container = self.table.item(self.filelist.index(command), 1)
+            container.add_to_space(space)
 
     def remove(self, filename):
-        table_filenames = list(self.table.cellWidget(index, 0).filename for index in range(self.table.rowCount()))
-        for index, label in enumerate(table_filenames):
-            if filename == label:
-                self.table.removeRow(index)
+        self.table.removeRow(self.filelist.index(filename))
         self.select()
         print 'removed: {0}'.format(filename)
 
     def select(self):
-        self.selection = []
-        axeslist = []
-        for index in range(self.table.rowCount()):
-            checkbox = self.table.cellWidget(index, 0)
-            if checkbox.checkState():
-                self.selection.append(checkbox.filename)
-                if checkbox.space == None:
-                    axeslist.append(BINoculars.space.Axes.fromfile(checkbox.filename))
-                else:
-                    axeslist.append(checkbox.space.axes)
+        axes = self.plotaxes
+        if len(axes) > 0:
+                self.emit(QtCore.SIGNAL('plotaxesChanged'), axes)
+        else:
+            self.emit(QtCore.SIGNAL('selectionError'), 'no spaces selected or spaces with non identical labels selected')
 
-        if len(self.selection) > 0:
-            first = axeslist[0]
-            if all(set(ax.label for ax in first) == set(ax.label for ax in axes) for axes in axeslist):
-                self.plotaxes = BINoculars.space.Axes(tuple(BINoculars.space.union_unequal_axes(ax[i] for ax in axeslist) for i in range(first.dimension)))
-                self.emit(QtCore.SIGNAL('plotaxesChanged'), self.plotaxes)
-            else:
-                self.selection = []
-                self.emit(QtCore.SIGNAL('selectionError'), 'labels of selected spaces not matching')
+    @property
+    def selection(self):
+        return list(container.label for checkbox, container in zip(self.itercheckbox(), self.itercontainer()) if checkbox.checkState())
+
+    @property
+    def plotaxes(self):
+        axes = tuple(container.get_ax() for checkbox, container in zip(self.itercheckbox(), self.itercontainer()) if checkbox.checkState())
+        if len(axes) > 0:
+            try:
+                return BINoculars.space.Axes(BINoculars.space.union_unequal_axes(ax) for ax in zip(*axes))
+            except ValueError:
+                return ()
         else:
-            self.emit(QtCore.SIGNAL('selectionError'), 'no spaces selected')
+            return ()
 
     @property
     def filelist(self):
-        return list(self.table.cellWidget(index, 0).filename for index in range(self.table.rowCount()))
+        return list(container.label for container in self.itercontainer())
 
     def getax(self, filename):
         index = self.filelist.index(filename)
-        checkbox = self.table.cellWidget(index, 0)
-        if checkbox.space == None:
-           return BINoculars.space.Axes.fromfile(checkbox.filename)
-        else:
-           return checkbox.space.axes
+        return self.table.item(index, 1).get_ax()
 
     def getspace(self, filename, key = None):
         index = self.filelist.index(filename)
-        checkbox = self.table.cellWidget(index, 0)
-        if checkbox.space == None:
-           return BINoculars.space.Space.fromfile(checkbox.filename, key = key)
-        else:
-           if key == None:
-                key = Ellipsis
-           return checkbox.space[key]
+        return self.table.item(index, 1).get_space(key)
+
+    def itercheckbox(self):
+        return iter(self.table.cellWidget(index, 0) for index in range(self.table.rowCount()))
 
+    def itercontainer(self):
+        return iter(self.table.item(index, 1) for index in range(self.table.rowCount()))
 
 class LimitWidget(QtGui.QWidget):
     def __init__(self, axes, parent=None):
         super(LimitWidget, self).__init__(parent)
-
         self.initUI(axes)
 
     def initUI(self, axes):
@@ -1209,6 +1268,8 @@ def is_empty(key):
 if __name__ == '__main__':
     app = QtGui.QApplication(sys.argv)
 
+    BINoculars.space.silence_numpy_errors()
+
     main = Window()
     main.resize(1000, 600)
     main.newproject()
diff --git a/processgui.py b/processgui.py
index e2a0e27..48da6f8 100755
--- a/processgui.py
+++ b/processgui.py
@@ -159,7 +159,7 @@ class Window(QtGui.QMainWindow):
 #----------------------------------------------------------------------------------------------------
 #-----------------------------------------CREATE TABLE-----------------------------------------------
 class Table(QtGui.QWidget):
-    def __init__(self, parent = None):
+    def __init__(self, label,  parent = None):
         super(Table, self).__init__()
         
         # create a QTableWidget
@@ -167,7 +167,7 @@ class Table(QtGui.QWidget):
         self.table.setHorizontalHeaderLabels(['Parameter', 'Value','Comment'])
         self.table.horizontalHeader().setStretchLastSection(True)
         self.table.verticalHeader().setVisible(False)
-        
+        self.table.setTextElideMode(QtCore.Qt.ElideLeft)
         #create combobox
         self.combobox = QtGui.QComboBox()
         #add items
@@ -179,14 +179,18 @@ class Table(QtGui.QWidget):
         self.connect(self.btn_add_row, QtCore.SIGNAL('clicked()'), self.add_row)
         self.buttonRemove = QtGui.QPushButton('-',self)
         self.connect(self.buttonRemove, QtCore.SIGNAL("clicked()"), self.remove)
-        self.btn_add_row.resize(10,10)
-        self.buttonRemove.resize(10,10) 
         #the dispositon of the table and the butttons
-        layout = QtGui.QGridLayout()
-        layout.addWidget(self.table,1,0,1,0)
-        layout.addWidget(self.btn_add_row,0,0)
-        layout.addWidget(self.buttonRemove,0,1)
-        self.setLayout(layout)
+
+        vbox = QtGui.QVBoxLayout()
+        hbox = QtGui.QHBoxLayout()
+
+        hbox.addWidget(self.btn_add_row)
+        hbox.addWidget(self.buttonRemove)
+
+        vbox.addWidget(label)
+        vbox.addLayout(hbox)
+        vbox.addWidget(self.table)
+        self.setLayout(vbox)
 
     def add_row(self):
         self.table.insertRow(self.table.rowCount())
@@ -261,14 +265,9 @@ class Conf_Tab(QtGui.QWidget):
 
         super(Conf_Tab,self).__init__()
         #we create 3 tables
-        self.Dis = Table()
-        self.Inp = Table()
-        self.Pro = Table()
-
-        label1 = QtGui.QLabel('<strong>Dispatcher :</strong>')
-        label2 = QtGui.QLabel('<strong>Input :</strong>')
-        label3 = QtGui.QLabel('<strong>Projection :<strong>')
-
+        self.Dis = Table(QtGui.QLabel('<strong>Dispatcher :</strong>'))
+        self.Inp = Table(QtGui.QLabel('<strong>Input :</strong>'))
+        self.Pro = Table(QtGui.QLabel('<strong>Projection :<strong>'))
         self.select = QtGui.QComboBox()
         backends = list(backend.lower() for backend in BINoculars.util.get_backends())
         #we add the list of different backends on the select combobox
@@ -278,18 +277,34 @@ class Conf_Tab(QtGui.QWidget):
         self.scan = QtGui.QLineEdit()
         self.scan.setToolTip('scan selection example: 820 824')
 
+        vbox = QtGui.QVBoxLayout()
+        hbox = QtGui.QHBoxLayout() 
+        splitter = QtGui.QSplitter(QtCore.Qt.Horizontal)
+        splitter.addWidget(self.Dis)
+        splitter.addWidget(self.Inp)
+        splitter.addWidget(self.Pro)
+        hbox.addWidget(splitter) 
+
+        commandbox = QtGui.QHBoxLayout() 
+        commandbox.addWidget(self.add)
+        commandbox.addWidget(self.scan)
+
+        vbox.addWidget(self.select)
+        vbox.addLayout(hbox)
+        vbox.addLayout(commandbox)
+
         #the dispositon of all elements of the gui
-        Layout = QtGui.QGridLayout()
-        Layout.addWidget(label1,1,1,1,2)
-        Layout.addWidget(label2,1,0,1,2)
-        Layout.addWidget(label3,1,2,1,2)
-        Layout.addWidget(self.select,0,0)
-        Layout.addWidget(self.Dis,2,1)
-        Layout.addWidget(self.Inp,2,0)
-        Layout.addWidget(self.Pro,2,2) 
-        Layout.addWidget(self.add,3,0)
-        Layout.addWidget(self.scan,3,1)
-        self.setLayout(Layout)
+        #Layout = QtGui.QGridLayout()
+        #Layout.addWidget(label1,1,1,1,2)
+        #Layout.addWidget(label2,1,0,1,2)
+        #Layout.addWidget(label3,1,2,1,2)
+        #Layout.addWidget(self.select,0,0)
+        #Layout.addWidget(self.Dis,2,1)
+        #Layout.addWidget(self.Inp,2,0)
+        #Layout.addWidget(self.Pro,2,2) 
+        #Layout.addWidget(self.add,3,0)
+        #Layout.addWidget(self.scan,3,1)
+        self.setLayout(vbox)
         
         #Here we call all methods for selected an ellement on differents combobox 
         self.Dis.add_to_combo(QtCore.QStringList(BINoculars.util.get_dispatchers()))
diff --git a/server.py b/server.py
index d8ec454..91ec1ea 100644
--- a/server.py
+++ b/server.py
@@ -104,7 +104,6 @@ if __name__ == '__main__':
     server.q = q
     ip, port = server.server_address
 
-    # interrupt the program with Ctrl-C
     print 'Process server started running at ip {0} and port {1}. Interrupt server with Ctrl-C'.format(ip, port)
     try:
         server.serve_forever()