Integration of TestData into testing pipeline.

conftest.py

@@ -0,0 +1,45 @@
+import pytest
+import requests
+import shutil
+from pathlib import Path
+
+
+def _download_and_extract_asset(tmp_dir_path: Path, asset_url: str):
+    asset_file_path = tmp_dir_path / 'asset.tar.gz'
+    response = requests.get(asset_url, stream=True)
+    if response.status_code == 200:
+        with asset_file_path.open(mode='wb') as f:
+            f.write(response.raw.read())
+    shutil.unpack_archive(asset_file_path, tmp_dir_path)
+
+
+@pytest.fixture(scope='session')
+def analysis_cpout_images_path(tmp_path_factory):
+    tmp_dir_path: Path = tmp_path_factory.mktemp('analysis_cpout_images')
+    _download_and_extract_asset(tmp_dir_path, 'https://github.com/BodenmillerGroup/TestData/releases/download/v1.0.1/210308_ImcTestData_analysis_cpout_images.tar.gz')
+    yield tmp_dir_path / 'datasets' / '210308_ImcTestData' / 'analysis' / 'cpout' / 'images'
+    shutil.rmtree(tmp_dir_path)
+
+
+@pytest.fixture(scope='session')
+def analysis_cpout_masks_path(tmp_path_factory):
+    tmp_dir_path: Path = tmp_path_factory.mktemp('analysis_cpout_images')
+    _download_and_extract_asset(tmp_dir_path, 'https://github.com/BodenmillerGroup/TestData/releases/download/v1.0.1/210308_ImcTestData_analysis_cpout_masks.tar.gz')
+    yield tmp_dir_path / 'datasets' / '210308_ImcTestData' / 'analysis' / 'cpout' / 'masks'
+    shutil.rmtree(tmp_dir_path)
+
+
+@pytest.fixture(scope='session')
+def analysis_ometiff_path(tmp_path_factory):
+    tmp_dir_path: Path = tmp_path_factory.mktemp('analysis_ometiff')
+    _download_and_extract_asset(tmp_dir_path, 'https://github.com/BodenmillerGroup/TestData/releases/download/v1.0.1/210308_ImcTestData_analysis_ometiff.tar.gz')
+    yield tmp_dir_path / 'datasets' / '210308_ImcTestData' / 'analysis' / 'ometiff'
+    shutil.rmtree(tmp_dir_path)
+
+
+@pytest.fixture(scope='session')
+def raw_path(tmp_path_factory):
+    tmp_dir_path: Path = tmp_path_factory.mktemp('raw')
+    _download_and_extract_asset(tmp_dir_path, 'https://github.com/BodenmillerGroup/TestData/releases/download/v1.0.1/210308_ImcTestData_raw.tar.gz')
+    yield tmp_dir_path / 'datasets' / '210308_ImcTestData' / 'raw'
+    shutil.rmtree(tmp_dir_path)

imctools/io/mcd/mcdxmlparser.py

@@ -4,7 +4,6 @@
 
 import xmltodict
 from dateutil.parser import parse
-from packaging import version
 
 import imctools.io.mcd.constants as const
 from imctools import __version__

tests/test_imcparser.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+from imctools.io.imc.imcparser import ImcParser
+
+
+class TestImcParser:
+    def test_read_imc_folder(self, analysis_ometiff_path: Path):
+        imc_folder_path = analysis_ometiff_path / '20210305_NE_mockData1'
+        parser = ImcParser(imc_folder_path)
+        ac_data = parser.get_acquisition_data(1)
+        assert parser.origin == "imc"
+        assert ac_data.is_valid is True
+        assert ac_data.image_data.shape == (5, 60, 60)
+        assert ac_data.n_channels == 5
+        assert ac_data.channel_names == ['Ag107', 'Pr141', 'Sm147', 'Eu153', 'Yb172']
+        assert ac_data.channel_labels == ['107Ag', 'Cytoker_651((3356))Pr141', 'Laminin_681((851))Sm147', 'YBX1_2987((3532))Eu153', 'H3K27Ac_1977((2242))Yb172']
+        assert ac_data.channel_masses == ['107', '141', '147', '153', '172']
+
+    def test_read_img(self, analysis_ometiff_path: Path):
+        imc_folder_path = analysis_ometiff_path / '20210305_NE_mockData1'
+        parser = ImcParser(imc_folder_path)
+        ac_data = parser.get_acquisition_data(1)
+        img = ac_data.get_image_by_name('Ag107')
+        assert img.shape == (60, 60)

tests/test_mcdparser.py

@@ -1,66 +1,22 @@
-import os
-import sys
-import zipfile
-from urllib.request import urlretrieve
+import pytest
+from pathlib import Path
 
-from tests.helpers import ParseTestMCD
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-
-TEST_DATA_URL = 'https://dl.dropboxusercontent.com/s/d8jk9zpi6rzcmsf/testdata_v2.1.zip'
-TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), 'testdata')
-TEST_ACQUISITIONS = os.path.join(TEST_DATA_FOLDER, 'acquisitions')
-TEST_RESULTS = os.path.join(TEST_DATA_FOLDER, 'test_results')
-EXT_MCD = '.mcd'
-EXT_RESULTS = EXT_MCD + '.pickle'
+from imctools.io.mcd.mcdparser import McdParser
 
 
 class TestMcdParser:
-    """ Compare the current MCD parser results with some stored ones """
-
-    def test_parser(self):
-        test_cases = TestMcdParser._get_test_cases()
-        for fn_mcd, testresults in test_cases.items():
-            testpickle = ParseTestMCD(testresults)
-            testpickle.load_test_dict_pickle(testresults)
-
-            testmcd = ParseTestMCD(fn_mcd)
-            testmcd.read_mcd(fn_mcd)
-            dict_p = testpickle.testdict
-            dict_m = testmcd.testdict
-            assert set(dict_p['acquisition_ids']) == set(dict_m['acquisition_ids'])
-            for ac in dict_p['acquisition_ids']:
-                ac_p = dict_p['acquisitions'][ac]
-                ac_m = dict_m['acquisitions'][ac]
-                for a in ['ac_desc', 'ac_rawdim', 'ac_nchan']:
-                    assert ac_p[a] == ac_m[a]
-                a = 'ac_channels'
-                assert str(ac_p[a]) == str(ac_m[a])
-
-    @staticmethod
-    def _get_test_cases():
-        if not os.path.exists(TEST_DATA_FOLDER):
-            os.makedirs(TEST_DATA_FOLDER)
-        zip_file = os.path.join(TEST_DATA_FOLDER, 'testdata_v2.1.zip')
-        if not os.path.isfile(zip_file):
-            urlretrieve(TEST_DATA_URL, zip_file)
-
-        with zipfile.ZipFile(zip_file, 'r') as zip_ref:
-            zip_ref.extractall(TEST_DATA_FOLDER)
-
-        fns_pickles = [f for f in os.listdir(TEST_RESULTS) if
-                       f.endswith(EXT_RESULTS)]
-        paths_mcd = []
-        for root, dirs, files in os.walk(TEST_ACQUISITIONS):
-            for f in files:
-                if f.endswith(EXT_MCD):
-                    paths_mcd.append(os.path.join(root, f))
-        test_cases = {p_mcd: os.path.join(TEST_RESULTS, f_pick)
-                     for p_mcd in paths_mcd
-                     for f_pick in fns_pickles if os.path.basename(p_mcd) in f_pick}
-        return test_cases
-
-
-if __name__ == '__main__':
-    parser = TestMcdParser()
-    parser.test_parser()
+    def test_read_invalid_suffix(self):
+        with pytest.raises(FileNotFoundError):
+            McdParser('file.unsupported_suffix')
+
+    def test_read_imc_mcd(self, raw_path: Path):
+        mcd_file_path = raw_path / '20210305_NE_mockData1' / '20210305_NE_mockData1.mcd'
+        parser = McdParser(mcd_file_path)
+        ac_data = parser.get_acquisition_data(1)
+        assert parser.origin == "mcd"
+        assert ac_data.is_valid is True
+        assert ac_data.image_data.shape == (5, 60, 60)
+        assert ac_data.n_channels == 5
+        assert ac_data.channel_names == ['Ag107', 'Pr141', 'Sm147', 'Eu153', 'Yb172']
+        assert ac_data.channel_labels == ['107Ag', 'Cytoker_651((3356))Pr141', 'Laminin_681((851))Sm147', 'YBX1_2987((3532))Eu153', 'H3K27Ac_1977((2242))Yb172']
+        assert ac_data.channel_masses == ['107', '141', '147', '153', '172']

tests/test_mcdxmlparser.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+from imctools.io.mcd.mcdparser import McdParser, McdXmlParser
+
+
+class TestMcdXmlParser:
+    def test_read_imc_mcd(self, raw_path: Path):
+        mcd_file_path = raw_path / '20210305_NE_mockData1' / '20210305_NE_mockData1.mcd'
+        mcd_parser = McdParser(mcd_file_path)
+        xml = mcd_parser.get_mcd_xml()
+        mcd_xml_parser = McdXmlParser(xml, str(mcd_file_path))
+        assert mcd_xml_parser.session.name == "20210305_NE_mockData1"

tests/test_ometiffparser.py

@@ -0,0 +1,22 @@
+import pytest
+from pathlib import Path
+
+from imctools.io.ometiff.ometiffparser import OmeTiffParser
+
+
+class TestOmeTiffParser:
+    def test_read_invalid_suffix(self):
+        with pytest.raises(FileNotFoundError):
+            OmeTiffParser('file.unsupported_suffix')
+
+    def test_read_ometiff(self, analysis_ometiff_path: Path):
+        ometiff_file_path = analysis_ometiff_path / '20210305_NE_mockData1' / '20210305_NE_mockData1_s0_a1_ac.ome.tiff'
+        parser = OmeTiffParser(ometiff_file_path)
+        ac_data = parser.get_acquisition_data()
+        assert parser.origin == "ome.tiff"
+        assert ac_data.is_valid is True
+        assert ac_data.image_data.shape == (5, 60, 60)
+        assert ac_data.n_channels == 5
+        assert ac_data.channel_names == ['Ag107', 'Pr141', 'Sm147', 'Eu153', 'Yb172']
+        assert ac_data.channel_labels == ['107Ag', 'Cytoker_651((3356))Pr141', 'Laminin_681((851))Sm147', 'YBX1_2987((3532))Eu153', 'H3K27Ac_1977((2242))Yb172']
+        assert ac_data.channel_masses == ['107', '141', '147', '153', '172']

tests/test_session.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from imctools.data import Session
+
+
+class TestSession:
+    def test_read_session(self, analysis_ometiff_path: Path):
+        session_file_path = analysis_ometiff_path / '20210305_NE_mockData1' / '20210305_NE_mockData1_session.json'
+        session = Session.load(session_file_path)
+        assert session.name == "20210305_NE_mockData1"
+        assert session.imctools_version == "2.1.4"
+        assert session.id == "fea546d5-03dd-42ea-9871-eef63e9d1c79"
+        assert list(session.slides.keys()) == [0]
+        assert list(session.acquisitions.keys()) == [1, 2, 3]
+        assert list(session.panoramas.keys()) == [1, 2]

tests/test_txtparser.py

@@ -0,0 +1,22 @@
+import pytest
+from pathlib import Path
+
+from imctools.io.txt.txtparser import TxtParser
+
+
+class TestTxtParser:
+    def test_read_invalid_suffix(self):
+        with pytest.raises(FileNotFoundError):
+            TxtParser('file.unsupported_suffix')
+
+    def test_read_ometiff(self, raw_path: Path):
+        txt_file_path = raw_path / '20210305_NE_mockData1' / '20210305_NE_mockData1_ROI_001_1.txt'
+        parser = TxtParser(txt_file_path)
+        ac_data = parser.get_acquisition_data()
+        assert parser.origin == "txt"
+        assert ac_data.is_valid is True
+        assert ac_data.image_data.shape == (5, 60, 60)
+        assert ac_data.n_channels == 5
+        assert ac_data.channel_names == ['Ag107', 'Pr141', 'Sm147', 'Eu153', 'Yb172']
+        assert ac_data.channel_labels == ['107Ag(Ag107Di)', 'Cytoker_651((3356))Pr141(Pr141Di)', 'Laminin_681((851))Sm147(Sm147Di)', 'YBX1_2987((3532))Eu153(Eu153Di)', 'H3K27Ac_1977((2242))Yb172(Yb172Di)']
+        assert ac_data.channel_masses == ['107', '141', '147', '153', '172']