Option to force usage of TXT files in mcdfolder_to_imcfolder when d…

…ealing with partially corrupted MCD files.

CHANGELOG.md

@@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.1.3] - 2021-01-21
+- Option to force usage of TXT files in `mcdfolder_to_imcfolder` when dealing with partially corrupted MCD files 
+
 ## [2.1.2] - 2021-01-21
 - Prevent crash on corrupted MCD acquisitions in order to keep valid ones
 

README.md

@@ -32,7 +32,7 @@ Version 1.x documentation (deprecated): https://imctools.readthedocs.io
 
 Preferable way to install `imctools` is via official PyPI registry. Please define package version explicitly in order to avoid incompatibilities between v1.x and v2.x versions:
 ```
-pip install imctools==2.1.2
+pip install imctools==2.1.3
 ```
 In old IMC segmentation pipelines versions 1.x should be used!
 ```

docs/cli.html

@@ -67,7 +67,7 @@ <h1 class="title">Module <code>imctools.cli</code></h1>
 
 def _add_mcdfolder2imcfolder_parser(subparsers: argparse._SubParsersAction):
     def func(args):
-        mcdfolder_to_imcfolder(args.input, args.output_folder, args.zip)
+        mcdfolder_to_imcfolder(args.input, args.output_folder, args.zip, args.parse_txt)
 
     parser = subparsers.add_parser(
         &#34;mcdfolder-to-imcfolder&#34;,
@@ -77,6 +77,7 @@ <h1 class="title">Module <code>imctools.cli</code></h1>
     parser.add_argument(&#34;input&#34;, help=&#34;Path to the folder/zip archive containing the IMC raw data.&#34;)
     parser.add_argument(&#34;output_folder&#34;, help=&#34;Path to the output folder.&#34;)
     parser.add_argument(&#34;--zip&#34;, action=&#34;store_true&#34;, help=&#34;Whether to create an output as .zip file.&#34;)
+    parser.add_argument(&#34;--parse_txt&#34;, action=&#34;store_true&#34;, help=&#34;Always use TXT files if present to get acquisition image data.&#34;)
     parser.set_defaults(func=func)
 
 

docs/converters/mcdfolder2imcfolder.html

@@ -41,7 +41,7 @@ <h1 class="title">Module <code>imctools.converters.mcdfolder2imcfolder</code></h
 logger = logging.getLogger(__name__)
 
 
-def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Path], create_zip: bool = False):
+def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Path], create_zip: bool = False, parse_txt: bool = False):
     &#34;&#34;&#34;Converts folder (or zipped folder) containing raw acquisition data (mcd and txt files) to IMC folder containing standardized files.
 
     Parameters
@@ -52,6 +52,8 @@ <h1 class="title">Module <code>imctools.converters.mcdfolder2imcfolder</code></h
         Path to the output folder.
     create_zip
         Whether to create an output as .zip file.
+    parse_txt
+        Always use TXT files if present to get acquisition image data.
     &#34;&#34;&#34;
     if isinstance(input, str):
         input = Path(input)
@@ -83,7 +85,7 @@ <h1 class="title">Module <code>imctools.converters.mcdfolder2imcfolder</code></h
         txt_files = glob.glob(str(input_folder / f&#34;*[0-9]{TXT_FILE_EXTENSION}&#34;))
         txt_acquisitions_map = {TxtParser.extract_acquisition_id(f): f for f in txt_files}
 
-        imc_writer = ImcWriter(output_folder, mcd_parser, txt_acquisitions_map)
+        imc_writer = ImcWriter(output_folder, mcd_parser, txt_acquisitions_map, parse_txt)
         imc_writer.write_imc_folder(create_zip=create_zip)
     finally:
         if mcd_parser is not None:
@@ -99,12 +101,15 @@ <h1 class="title">Module <code>imctools.converters.mcdfolder2imcfolder</code></h
 
     mcdfolder_to_imcfolder(
         # Path(&#34;/home/anton/Downloads/20170905_Fluidigmworkshopfinal_SEAJa.zip&#34;),
-        &#34;/home/anton/Downloads/20170905_Fluidigmworkshopfinal_SEAJa&#34;,
+        # &#34;/home/anton/Downloads/20170905_Fluidigmworkshopfinal_SEAJa&#34;,
+        &#34;/home/anton/Data/forAnton_MCD/6505_Lympho&#34;,
         # &#34;/home/anton/Documents/merrick/IMC/20200904_MS_XRF_epithelial_panel_4_titration.zip&#34;,
         # &#34;/home/anton/Documents/IMC Workshop 2019/Data/iMC_workshop_2019/20190919_FluidigmBrCa_SE&#34;,
         # &#34;/home/anton/Downloads/test&#34;,
         # &#34;/home/anton/Data/ForAnton/20200123_IMMUcan_reproducibility_day1_sl1_cp_panel_1_1.06.zip&#34;,
         Path(&#34;/home/anton/Downloads/imc_folder_v2&#34;),
+        create_zip=False,
+        parse_txt=True
     )
 
     print(timeit.default_timer() - tic)</code></pre>
@@ -118,7 +123,7 @@ <h1 class="title">Module <code>imctools.converters.mcdfolder2imcfolder</code></h
 <h2 class="section-title" id="header-functions">Functions</h2>
 <dl>
 <dt id="imctools.converters.mcdfolder2imcfolder.mcdfolder_to_imcfolder"><code class="name flex">
-<span>def <span class="ident">mcdfolder_to_imcfolder</span></span>(<span>input: Union[str, pathlib.Path], output_folder: Union[str, pathlib.Path], create_zip: bool = False)</span>
+<span>def <span class="ident">mcdfolder_to_imcfolder</span></span>(<span>input: Union[str, pathlib.Path], output_folder: Union[str, pathlib.Path], create_zip: bool = False, parse_txt: bool = False)</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Converts folder (or zipped folder) containing raw acquisition data (mcd and txt files) to IMC folder containing standardized files.</p>
@@ -131,12 +136,14 @@ <h2 id="parameters">Parameters</h2>
 <dd>Path to the output folder.</dd>
 <dt><strong><code>create_zip</code></strong></dt>
 <dd>Whether to create an output as .zip file.</dd>
+<dt><strong><code>parse_txt</code></strong></dt>
+<dd>Always use TXT files if present to get acquisition image data.</dd>
 </dl></div>
 <details class="source">
 <summary>
 <span>Expand source code</span>
 </summary>
-<pre><code class="python">def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Path], create_zip: bool = False):
+<pre><code class="python">def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Path], create_zip: bool = False, parse_txt: bool = False):
     &#34;&#34;&#34;Converts folder (or zipped folder) containing raw acquisition data (mcd and txt files) to IMC folder containing standardized files.
 
     Parameters
@@ -147,6 +154,8 @@ <h2 id="parameters">Parameters</h2>
         Path to the output folder.
     create_zip
         Whether to create an output as .zip file.
+    parse_txt
+        Always use TXT files if present to get acquisition image data.
     &#34;&#34;&#34;
     if isinstance(input, str):
         input = Path(input)
@@ -178,7 +187,7 @@ <h2 id="parameters">Parameters</h2>
         txt_files = glob.glob(str(input_folder / f&#34;*[0-9]{TXT_FILE_EXTENSION}&#34;))
         txt_acquisitions_map = {TxtParser.extract_acquisition_id(f): f for f in txt_files}
 
-        imc_writer = ImcWriter(output_folder, mcd_parser, txt_acquisitions_map)
+        imc_writer = ImcWriter(output_folder, mcd_parser, txt_acquisitions_map, parse_txt)
         imc_writer.write_imc_folder(create_zip=create_zip)
     finally:
         if mcd_parser is not None:

docs/index.html

@@ -49,7 +49,7 @@ <h2 id="prerequisites">Prerequisites</h2>
 </ul>
 <h2 id="installation">Installation</h2>
 <p>Preferable way to install <code><a title="imctools" href="#imctools">imctools</a></code> is via official PyPI registry. Please define package version explicitly in order to avoid incompatibilities between v1.x and v2.x versions:</p>
-<pre><code>pip install imctools==2.1.2
+<pre><code>pip install imctools==2.1.3
 </code></pre>
 <p>In old IMC segmentation pipelines versions 1.x should be used!</p>
 <pre><code>pip install imctools==1.0.8
@@ -140,7 +140,7 @@ <h3 id="usage-of-previous-version-1x">Usage of previous version 1.x</h3>
 
 __author__ = &#34;&#34;&#34;Vito Zanotelli, Anton Rau&#34;&#34;&#34;
 __email__ = &#34;[email protected], [email protected]&#34;
-__version__ = &#34;2.1.1&#34;</code></pre>
+__version__ = &#34;2.1.3&#34;</code></pre>
 </details>
 </section>
 <section>

docs/io/imc/imcwriter.html

@@ -50,6 +50,7 @@ <h1 class="title">Module <code>imctools.io.imc.imcwriter</code></h1>
         root_output_folder: Union[str, Path],
         mcd_parser: McdParser,
         txt_acquisitions_map: Dict[int, Union[str, Path]] = None,
+        parse_txt: bool = False
     ):
         &#34;&#34;&#34;
         Initializes an ImcFolderWriter that can be used to write out an imcfolder and compress it to zip.
@@ -59,6 +60,7 @@ <h1 class="title">Module <code>imctools.io.imc.imcwriter</code></h1>
         self.root_output_folder = root_output_folder
         self.mcd_parser = mcd_parser
         self.txt_acquisitions_map = txt_acquisitions_map
+        self.parse_txt = parse_txt
 
     @property
     def folder_name(self):
@@ -84,7 +86,7 @@ <h1 class="title">Module <code>imctools.io.imc.imcwriter</code></h1>
         # Save acquisition images in OME-TIFF format
         for acquisition in session.acquisitions.values():
             acquisition_data = self.mcd_parser.get_acquisition_data(acquisition.id)
-            if not acquisition_data.is_valid:
+            if self.parse_txt or not acquisition_data.is_valid:
                 if self.txt_acquisitions_map is not None and acquisition.id in self.txt_acquisitions_map:
                     logger.warning(f&#34;Using TXT file for acquisition: {acquisition.id}&#34;)
                     try:
@@ -161,7 +163,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
 <dl>
 <dt id="imctools.io.imc.imcwriter.ImcWriter"><code class="flex name class">
 <span>class <span class="ident">ImcWriter</span></span>
-<span>(</span><span>root_output_folder: Union[str, pathlib.Path], mcd_parser: <a title="imctools.io.mcd.mcdparser.McdParser" href="../mcd/mcdparser.html#imctools.io.mcd.mcdparser.McdParser">McdParser</a>, txt_acquisitions_map: Dict[int, Union[str, pathlib.Path]] = None)</span>
+<span>(</span><span>root_output_folder: Union[str, pathlib.Path], mcd_parser: <a title="imctools.io.mcd.mcdparser.McdParser" href="../mcd/mcdparser.html#imctools.io.mcd.mcdparser.McdParser">McdParser</a>, txt_acquisitions_map: Dict[int, Union[str, pathlib.Path]] = None, parse_txt: bool = False)</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Write IMC session data to IMC folder structure.</p>
@@ -178,6 +180,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
         root_output_folder: Union[str, Path],
         mcd_parser: McdParser,
         txt_acquisitions_map: Dict[int, Union[str, Path]] = None,
+        parse_txt: bool = False
     ):
         &#34;&#34;&#34;
         Initializes an ImcFolderWriter that can be used to write out an imcfolder and compress it to zip.
@@ -187,6 +190,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
         self.root_output_folder = root_output_folder
         self.mcd_parser = mcd_parser
         self.txt_acquisitions_map = txt_acquisitions_map
+        self.parse_txt = parse_txt
 
     @property
     def folder_name(self):
@@ -212,7 +216,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
         # Save acquisition images in OME-TIFF format
         for acquisition in session.acquisitions.values():
             acquisition_data = self.mcd_parser.get_acquisition_data(acquisition.id)
-            if not acquisition_data.is_valid:
+            if self.parse_txt or not acquisition_data.is_valid:
                 if self.txt_acquisitions_map is not None and acquisition.id in self.txt_acquisitions_map:
                     logger.warning(f&#34;Using TXT file for acquisition: {acquisition.id}&#34;)
                     try:
@@ -309,7 +313,7 @@ <h3>Methods</h3>
     # Save acquisition images in OME-TIFF format
     for acquisition in session.acquisitions.values():
         acquisition_data = self.mcd_parser.get_acquisition_data(acquisition.id)
-        if not acquisition_data.is_valid:
+        if self.parse_txt or not acquisition_data.is_valid:
             if self.txt_acquisitions_map is not None and acquisition.id in self.txt_acquisitions_map:
                 logger.warning(f&#34;Using TXT file for acquisition: {acquisition.id}&#34;)
                 try:

imctools/__init__.py

@@ -6,4 +6,4 @@
 
 __author__ = """Vito Zanotelli, Anton Rau"""
 __email__ = "[email protected], [email protected]"
-__version__ = "2.1.1"
+__version__ = "2.1.3"

imctools/cli.py

@@ -38,7 +38,7 @@ def main():
 
 def _add_mcdfolder2imcfolder_parser(subparsers: argparse._SubParsersAction):
     def func(args):
-        mcdfolder_to_imcfolder(args.input, args.output_folder, args.zip)
+        mcdfolder_to_imcfolder(args.input, args.output_folder, args.zip, args.parse_txt)
 
     parser = subparsers.add_parser(
         "mcdfolder-to-imcfolder",
@@ -48,6 +48,7 @@ def func(args):
     parser.add_argument("input", help="Path to the folder/zip archive containing the IMC raw data.")
     parser.add_argument("output_folder", help="Path to the output folder.")
     parser.add_argument("--zip", action="store_true", help="Whether to create an output as .zip file.")
+    parser.add_argument("--parse_txt", action="store_true", help="Always use TXT files if present to get acquisition image data.")
     parser.set_defaults(func=func)
 
 

imctools/converters/mcdfolder2imcfolder.py

@@ -13,7 +13,7 @@
 logger = logging.getLogger(__name__)
 
 
-def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Path], create_zip: bool = False):
+def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Path], create_zip: bool = False, parse_txt: bool = False):
     """Converts folder (or zipped folder) containing raw acquisition data (mcd and txt files) to IMC folder containing standardized files.
 
     Parameters
@@ -24,6 +24,8 @@ def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Pa
         Path to the output folder.
     create_zip
         Whether to create an output as .zip file.
+    parse_txt
+        Always use TXT files if present to get acquisition image data.
     """
     if isinstance(input, str):
         input = Path(input)
@@ -55,7 +57,7 @@ def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Pa
         txt_files = glob.glob(str(input_folder / f"*[0-9]{TXT_FILE_EXTENSION}"))
         txt_acquisitions_map = {TxtParser.extract_acquisition_id(f): f for f in txt_files}
 
-        imc_writer = ImcWriter(output_folder, mcd_parser, txt_acquisitions_map)
+        imc_writer = ImcWriter(output_folder, mcd_parser, txt_acquisitions_map, parse_txt)
         imc_writer.write_imc_folder(create_zip=create_zip)
     finally:
         if mcd_parser is not None:
@@ -71,12 +73,15 @@ def mcdfolder_to_imcfolder(input: Union[str, Path], output_folder: Union[str, Pa
 
     mcdfolder_to_imcfolder(
         # Path("/home/anton/Downloads/20170905_Fluidigmworkshopfinal_SEAJa.zip"),
-        "/home/anton/Downloads/20170905_Fluidigmworkshopfinal_SEAJa",
+        # "/home/anton/Downloads/20170905_Fluidigmworkshopfinal_SEAJa",
+        "/home/anton/Data/forAnton_MCD/6505_Lympho",
         # "/home/anton/Documents/merrick/IMC/20200904_MS_XRF_epithelial_panel_4_titration.zip",
         # "/home/anton/Documents/IMC Workshop 2019/Data/iMC_workshop_2019/20190919_FluidigmBrCa_SE",
         # "/home/anton/Downloads/test",
         # "/home/anton/Data/ForAnton/20200123_IMMUcan_reproducibility_day1_sl1_cp_panel_1_1.06.zip",
         Path("/home/anton/Downloads/imc_folder_v2"),
+        create_zip=False,
+        parse_txt=True
     )
 
     print(timeit.default_timer() - tic)

imctools/io/imc/imcwriter.py

@@ -22,6 +22,7 @@ def __init__(
         root_output_folder: Union[str, Path],
         mcd_parser: McdParser,
         txt_acquisitions_map: Dict[int, Union[str, Path]] = None,
+        parse_txt: bool = False
     ):
         """
         Initializes an ImcFolderWriter that can be used to write out an imcfolder and compress it to zip.
@@ -31,6 +32,7 @@ def __init__(
         self.root_output_folder = root_output_folder
         self.mcd_parser = mcd_parser
         self.txt_acquisitions_map = txt_acquisitions_map
+        self.parse_txt = parse_txt
 
     @property
     def folder_name(self):
@@ -56,7 +58,7 @@ def write_imc_folder(self, create_zip: bool = True, remove_folder: bool = None):
         # Save acquisition images in OME-TIFF format
         for acquisition in session.acquisitions.values():
             acquisition_data = self.mcd_parser.get_acquisition_data(acquisition.id)
-            if not acquisition_data.is_valid:
+            if self.parse_txt or not acquisition_data.is_valid:
                 if self.txt_acquisitions_map is not None and acquisition.id in self.txt_acquisitions_map:
                     logger.warning(f"Using TXT file for acquisition: {acquisition.id}")
                     try:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "imctools"
-version = "2.1.2"
+version = "2.1.3"
 description = "Tools to handle Fluidigm IMC data"
 license = "MIT"
 authors = ["Vito Zanotelli <[email protected]>", "Anton Rau <[email protected]>"]

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1.2
+current_version = 2.1.3
 commit = False
 tag = False