Merge pull request #34 from BodenmillerGroup/development

Makes the development branch the new master.

README.md

@@ -19,8 +19,15 @@ For a description of the associated segmentation pipline, please visit: https://
 * The core functions have a 'base' pure Python/Jython implementation with no dependencies outside the standard libraries.
 * The fast functions do need Python packages, such as numpy, scipy etc. installed.
 
+## Installation
+Use the pip installation manger to directly install the package from Github:
+
+```
+pip install git+https://github.com/BodenmillerGroup/imctools.git
+```
+
 ## Usage
-imctools is often used from jupyter as aprt of the preprocessing pipeline, mainly using the 'script' wrapper functions. Check 'notebooks/example_preprocessing_pipline.ipynb' as a template
+imctools is often used from jupyter as part of the preprocessing pipeline, mainly using the 'script' wrapper functions. Check 'notebooks/example_preprocessing_pipline.ipynb' as a template
 
 Further imctools can be directly used as a module:
 

imctools/io/__init__.py

@@ -0,0 +1,45 @@
+import warnings
+
+CHANGE_DTYPE_LB_WARNING = 'Data minimum trunkated as outside dtype range'
+CHANGE_DTYPE_UB_WARNING = 'Data max trunkated as outside dtype range'
+
+# Checks if numpy is available in the
+# Python implementation
+
+_is_base = False
+try:
+    import numpy as np
+except ImportError:
+    _is_base = True
+
+if _is_base == False:
+    def change_dtype(array, dtype):
+        """
+        Changes the dtype of an array
+
+        This makes sure that the values are correctly truncated and rounded
+        to fit into the new dtype.
+
+        :param array: a numpy array
+        :param dtypw: a numpy dtype
+        :returns: a copy of the array with the correct dtype.
+        """
+        if dtype.kind in ['i', 'u']:
+            dinf = np.iinfo(dtype)
+            array = np.around(array)
+            mina = array.min()
+            maxa = array.max()
+            t_min = None
+            t_max = None
+            if mina < dinf.min:
+                t_min = dinf.min
+                warnings.warn(CHANGE_DTYPE_LB_WARNING)
+
+            if maxa > dinf.max:
+                t_max = dinf.max
+                warnings.warn(CHANGE_DTYPE_UB_WARNING)
+            if (t_min is not None) | (t_max is not None):
+                # this can be done inplace, as np.around returns array new object.
+                np.clip(array, a_min=t_min, a_max=t_max, out=array)
+        array = array.astype(dtype)
+        return array

imctools/io/imcfolderwriter.py

@@ -26,7 +26,7 @@ def __init__(self, out_folder, mcddata=None, imcacquisitions=None, mcdmeta=None)
 
         if mcddata is not None:
             self.add_mcddata(mcddata, add_acquisitions=add_ac)
-        
+
         if self.meta is None:
             raise ValueError('At least mcdata or mcdmeta need to be specified!')
 
@@ -46,7 +46,10 @@ def add_mcddata(self, mcddata, add_acquisitions=True):
             imcacs = self.mcd.get_all_imcacquistions()
             self.add_imcacquisitions(imcacs)
 
-    def write_imc_folder(self, zipfolder=True, remove_folder=True):
+    def write_imc_folder(self, zipfolder=True, remove_folder=None):
+        if remove_folder is None:
+            remove_folder = zipfolder
+
         base_folder = self.out_folder
         foldername = self.foldername
         out_folder = os.path.join(self.out_folder, self.foldername)
@@ -57,6 +60,7 @@ def write_imc_folder(self, zipfolder=True, remove_folder=True):
             self._write_acquisition(ac, out_folder)
         if self.meta:
             self.meta.save_meta_xml(out_folder)
+            self.meta.save_meta_csv(out_folder)
 
         if self.mcd:
             slide_ids = self.meta.objects.get(mcdmeta.SLIDE, dict()).keys()
@@ -102,7 +106,7 @@ def _find_ac_metaname_from_txt_fn(self, ac):
 
 if __name__ == '__main__':
     import imctools.io.mcdparser as mcdp
-    #fn_mcd = '/home/vitoz/temp/txtvsmcd/20170805_p60-63_slide6_ac1_vz.mcd' 
+    #fn_mcd = '/home/vitoz/temp/txtvsmcd/20170805_p60-63_slide6_ac1_vz.mcd'
     #fn_mcd = '/mnt/imls-bod/VitoZ/Spheres/20161130_p25_slide2_ac1/20161130_p25_slide2_ac1.mcd'
     #fn_mcd='/mnt/imls-bod/VitoZ/Spheres/20161005_IS2362_4_site1_ac1/20161005_IS2362_4_site1_ac1.mcd'
     # an example of not functional mcd but working txt
@@ -113,4 +117,4 @@ def _find_ac_metaname_from_txt_fn(self, ac):
     ifw = ImcFolderWriter('/home/vitoz/temp/', mcddata=mcd)
     ifw.write_imc_folder()
 
-    
+

imctools/io/mcdxmlparser.py

@@ -2,6 +2,7 @@
 import imctools.librarybase as libb
 from collections import OrderedDict
 import os
+import csv
 
 """
 This module should help parsing the MCD xml metadata
@@ -77,13 +78,13 @@
 WIDTHUM = 'WidthUm'
 
 PARSER = 'parser'
-
+META_CSV = '_meta.csv'
 """
 Definition of all the meta objects
 Each entity will have a class corresponding to it, with helpermethods
 that e.g. allow to retrieve images etc.
 
-This is implemented as parent-child relationships where each entry has a list of parents 
+This is implemented as parent-child relationships where each entry has a list of parents
 and a nested dictionary of children of the form (child_type: childID: childobject)
 
 Further each object is registered in the global root node, making them easy accessible.
@@ -124,7 +125,7 @@ def __init__(self, mtype, meta, parents, symbol=None):
     @property
     def is_root(self):
        return len(self.parents) == 0
-    
+
     def _update_parents(self, p):
         self._update_dict(p.childs)
 
@@ -170,7 +171,7 @@ def __init__(self, meta, parents):
 
     def get_channels(self):
         return self.childs[ACQUISITIONCHANNEL]
-    
+
     def get_channel_orderdict(self):
         chan_dic = self.get_channels()
         out_dic = dict()
@@ -268,7 +269,8 @@ def _init_objects(self):
                 if len(parents) == 0:
                     parents = [self]
                 ObjClass(o, parents)
-
+
+
     def get_objects_by_id(self, idname, objid):
         """
         Gets objects by idname and id
@@ -278,7 +280,7 @@ def get_objects_by_id(self, idname, objid):
         """
         mtype = ID_DICT[idname]
         return self.get_object(mtype, objid)
-        
+
     def get_object(self, mtype, mid):
         """
         Return an object defined by type and id
@@ -292,7 +294,7 @@ def _get_meta_objects(self, mtype):
         """
         A helper to get objects, e.g. slides etc. metadata
         from the metadata dict. takes care of the case where
-        only one object is present and thus a dict and not a 
+        only one object is present and thus a dict and not a
         list of dicts is returned.
         """
         objs = self.properties.get(mtype)
@@ -306,6 +308,21 @@ def save_meta_xml(self, out_folder):
         et.ElementTree.ElementTree(xml).write(
             os.path.join(out_folder,fn), encoding='utf-8')
 
+    def save_meta_csv(self, out_folder):
+        """
+        Writes the xml data as csv tables
+
+        """
+        for n, o in self.objects.items():
+            odict = [i.properties for k, i in o.items()]
+            fn = '_'.join([self.metaname, n]) + META_CSV
+            with open(os.path.join(out_folder, fn), 'w') as csvfile:
+                cols = odict[0].keys()
+                writer = csv.DictWriter(csvfile, sorted(cols))
+                writer.writeheader()
+                for row in odict:
+                    writer.writerow(row)
+
     def get_channels(self):
         """
         gets a list of all channels
@@ -323,7 +340,7 @@ def get_acquisition_meta(self, acid):
         Returns the acquisition metadata dict
         """
         return self.get_object(ACQUISITION, acid).properties
-    
+
 
     def get_acquisition_rois(self):
         """

imctools/io/ometiffparser.py

@@ -42,8 +42,17 @@ def get_imc_acquisition(self):
 
     def read_image(self, filename):
         with tifffile.TiffFile(filename) as tif:
-            self.data = tif.asarray()
-            self.ome = tif.pages[0].tags['image_description'].value
+            try:
+                self.data = tif.asarray(out='memmap')
+            except:
+                # this is in an older tifffile version is used
+                self.data = tif.asarray()
+            try:
+                self.ome = tif.pages[0].tags['ImageDescription'].value
+            except:
+                self.ome = tif.pages[0].tags['image_description'].value
+
+
 
     # @staticmethod
     # def reshape_flat(data):
@@ -72,4 +81,4 @@ def read_image(self, filename):
     plt.imshow(np.array(imc_ac.get_img_stack_cyx([0])).squeeze())
     plt.show()
     print(imc_ac)
-    print(imc_ac.channel_metals)
+    print(imc_ac.channel_metals)

imctools/io/tiffwriter.py

@@ -2,10 +2,13 @@
 import numpy as np
 import tifffile
 import imctools.external.omexml as ome
-
 from xml.etree import cElementTree as ElementTree
-
 import sys
+
+import warnings
+
+from imctools.io import change_dtype, CHANGE_DTYPE_LB_WARNING, CHANGE_DTYPE_UB_WARNING
+
 if sys.version_info.major == 3:
     from io import StringIO
     uenc = 'unicode'
@@ -18,10 +21,25 @@ class TiffWriter(object):
     """
 
     """
-    pixeltype_dict = ({np.int64().dtype: ome.PT_INT16,
+    pixeltype_dict = ({np.int64().dtype: ome.PT_FLOAT,
+                       np.int32().dtype: ome.PT_INT32,
+                       np.int16().dtype: ome.PT_INT16,
+                       np.uint16().dtype: ome.PT_UINT16,
+                       np.uint32().dtype: ome.PT_UINT32,
+                       np.uint8().dtype: ome.PT_UINT8,
                        np.float32().dtype: ome.PT_FLOAT,
-                       np.float64().dtype: ome.PT_FLOAT})
-
+                       np.float64().dtype: ome.PT_DOUBLE
+                       })
+    pixeltype_np = {
+            ome.PT_FLOAT: np.dtype('float32'),
+            ome.PT_DOUBLE: np.dtype('float64'),
+            ome.PT_UINT8: np.dtype('uint8'),
+            ome.PT_UINT16: np.dtype('uint16'),
+            ome.PT_UINT32: np.dtype('uint32'),
+            ome.PT_INT8: np.dtype('int8'),
+            ome.PT_INT16: np.dtype('int16'),
+            ome.PT_INT32: np.dtype('int32')
+        }
     def __init__(self, file_name, img_stack, pixeltype =None, channel_name=None, original_description=None, fluor=None):
         self.file_name = file_name
         self.img_stack = img_stack
@@ -40,19 +58,21 @@ def __init__(self, file_name, img_stack, pixeltype =None, channel_name=None, ori
 
         self.original_description = original_description
 
-    def save_image(self, mode='imagej', compression=0, dtype=None, bigtiff=True):
+    def save_image(self, mode='imagej', compression=0, dtype=None, bigtiff=False):
         #TODO: add original metadata somehow
         fn_out = self.file_name
         img = self.img_stack.swapaxes(2, 0)
         if dtype is not None:
             dt = np.dtype(dtype)
-            img = img.astype(dt)
+        else:
+            dt = self.pixeltype_np[self.pixeltype]
+        img = change_dtype(img, dt)
         # img = img.reshape([1,1]+list(img.shape)).swapaxes(2, 0)
         if mode == 'imagej':
             tifffile.imsave(fn_out, img, compress=compression, imagej=True,
                             bigtiff=bigtiff)
         elif mode == 'ome':
-            xml = self.get_xml()
+            xml = self.get_xml(dtype=dtype)
             tifffile.imsave(fn_out, img, compress=compression, imagej=False,
                             description=xml, bigtiff=bigtiff)
 
@@ -74,7 +94,11 @@ def save_image(self, mode='imagej', compression=0, dtype=None, bigtiff=True):
     def nchannels(self):
         return self.img_stack.shape[2]
 
-    def get_xml(self):
+    def get_xml(self, dtype=None):
+        if dtype is not None:
+            pixeltype = self.pixeltype_dict[dtype]
+        else:
+            pixeltype = self.pixeltype
         img = self.img_stack
         omexml = ome.OMEXML()
         omexml.image(0).Name = os.path.basename(self.file_name)
@@ -85,7 +109,7 @@ def get_xml(self):
         p.SizeT = 1
         p.SizeZ = 1
         p.DimensionOrder = ome.DO_XYCZT
-        p.PixelType = self.pixeltype
+        p.PixelType = pixeltype
         p.channel_count = self.nchannels
         for i in range(self.nchannels):
             channel_info = self.channel_name[i]