primary-problem-study-rsna-2024.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71549,"databundleVersionId":8561470,"sourceType":"competition"},{"sourceId":8236972,"sourceType":"datasetVersion","datasetId":4885707},{"sourceId":8942685,"sourceType":"datasetVersion","datasetId":5380929}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install duckdb --no-index --find-links=file:///kaggle/input/polars-and-duckdb/kaggle/working/mysitepackages/duck_pkg","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-03T15:48:30.708082Z","iopub.execute_input":"2024-08-03T15:48:30.708961Z","iopub.status.idle":"2024-08-03T15:48:44.265034Z","shell.execute_reply.started":"2024-08-03T15:48:30.708919Z","shell.execute_reply":"2024-08-03T15:48:44.264145Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Looking in links: file:///kaggle/input/polars-and-duckdb/kaggle/working/mysitepackages/duck_pkg\nProcessing /kaggle/input/polars-and-duckdb/kaggle/working/mysitepackages/duck_pkg/duckdb-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\nInstalling collected packages: duckdb\nSuccessfully installed duckdb-0.8.1\n","output_type":"stream"}]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport os\nimport polars as pl\nimport duckdb as dd\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport cv2\nfrom pydicom import dcmread\nimport warnings\nfrom sklearn.preprocessing import LabelEncoder\nimport pickle\nimport gc\nimport ctypes","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:48:49.937422Z","iopub.execute_input":"2024-08-03T15:48:49.938267Z","iopub.status.idle":"2024-08-03T15:48:51.533174Z","shell.execute_reply.started":"2024-08-03T15:48:49.938233Z","shell.execute_reply":"2024-08-03T15:48:51.532244Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"markdown","source":"## Conditions\n### spinal_canal_stenosis\n### neural_foraminal_narrowing\n#### left/right\n### subarticular_stenosis\n#### left/right\n\n## Levels of vertebrae\n#### l1_l2\n#### l2_l3\n#### l3_l4\n#### l4_l5\n#### l5_s1\n\n> Each Study has series of images for these three conditions. The series description tells us what type of image is in the dcm file","metadata":{}},{"cell_type":"markdown","source":"### Reading the available training metadata files using polars\n### Bringing the metadata files in a desired shape","metadata":{}},{"cell_type":"code","source":"train_meta_data = pl.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train.csv'\\\n                              , low_memory=True)\ntrain_series_desc = pl.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_series_descriptions.csv'\\\n                                , low_memory=True)\ntrain_label_coordinates = pl.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_label_coordinates.csv'\\\n                                      , low_memory=True)\n\ntrain_meta_data_w_srs_desc = dd.sql(\"select b.series_description, b.series_id, a.* from train_meta_data a \\\nleft join train_series_desc b on a.study_id = b.study_id\").pl()\n\nmelt_cols = train_meta_data_w_srs_desc.columns[3:]\ntrain_meta_data_melted = train_meta_data_w_srs_desc.melt(id_vars=[\"study_id\",\"series_id\",\"series_description\"], value_vars=melt_cols)\ntrain_meta_data_melted.columns = ['study_id', 'series_id', 'series_description', 'condition_vert_level', 'severity']\n\ntrain_label_coordinates_formatted = dd.sql(\"select study_id, series_id, instance_number \\\n,lower(replace(condition,' ','_')||'_'||replace(level,'/','_')) as condition, x, y \\\nfrom train_label_coordinates\").pl()\n\nfull_training_data = dd.sql(\"select t1.study_id, t1.series_id, t1.series_description, t2.instance_number \\\n, t1.condition_vert_level, t1.severity, t2.x, t2.y \\\nfrom train_meta_data_melted t1 \\\nleft join train_label_coordinates_formatted t2 on t1.study_id = t2.study_id \\\nand t1.condition_vert_level = t2.condition \\\n\").pl()\n\nprint(full_training_data.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-20T07:15:35.021117Z","iopub.execute_input":"2024-07-20T07:15:35.021680Z","iopub.status.idle":"2024-07-20T07:15:35.434470Z","shell.execute_reply.started":"2024-07-20T07:15:35.021630Z","shell.execute_reply":"2024-07-20T07:15:35.433287Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"(157350, 8)\n","output_type":"stream"}]},{"cell_type":"code","source":"dd.sql(\"select * from train_meta_data_melted where study_id = 3008676218 \").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Checking a specific study/patient's scans and the corresponding details, only to get a better sense of the data","metadata":{}},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=100)\npl.Config.set_tbl_rows(1000)\n\n#print(train_meta_data_melted.shape)\n# print(train_meta_data.filter(pl.col('study_id')==100206310).transpose(include_header=True))\n#print(train_meta_data_melted.filter(pl.col('study_id')==100206310))\n# print(train_series_desc.filter(pl.col('study_id')==100206310))\n#print(train_label_coordinates_formatted.shape)\n#print(train_label_coordinates_formatted.filter(pl.col('study_id')==100206310))\n#print(train_series_coordinates.filter(pl.col('study_id')==100206310).sort(['study_id','series_id']))\nprint(full_training_data.filter(pl.col('study_id')==100206310).sort(['study_id','series_id']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"condition_list = dd.sql(\"select distinct condition_vert_level from full_training_data\").pl().to_series().to_list()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select series_description, severity, count(distinct(study_id||'/'||series_id)) as ss_count from full_training_data \\\nwhere condition_vert_level = 'left_neural_foraminal_narrowing_l1_l2' and severity is not null and severity != 'Severe' group by series_description, severity\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select series_description, count(distinct(study_id||'/'||series_id)) as ss_count from full_training_data \\\nwhere severity = 'Severe' group by series_description\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=100)\npl.Config.set_tbl_rows(1000)\n\ndd.sql(\"select condition_vert_level, series_description, count(distinct(study_id||'/'||series_id)) as ss_count from full_training_data \\\nwhere severity = 'Severe' group by condition_vert_level, series_description\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_path_root = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/'\n\nfor condition in condition_list:\n    print(\"generating feature metadata for {0}\".format(condition))\n\n    full_feature_metadata = dd.sql(\"select distinct ('{0}'||study_id||'/'||series_id) as file_path_root, severity from full_training_data \\\n    where condition_vert_level = '{1}' and severity is not null\".format(train_files_path_root, condition)).pl()\n\n    full_feature_metadata_pd = full_feature_metadata.to_pandas()\n\n    label_mapping = {}\n\n    for file_path in range(full_feature_metadata_pd.shape[0]):\n        study_series_path = full_feature_metadata_pd['file_path_root'][file_path]\n        for image_file_path in os.listdir(study_series_path):\n            full_img_path = os.path.join(study_series_path, image_file_path)\n            label_mapping[full_img_path] = full_feature_metadata_pd['severity'][file_path]\n\n\n    # Create a list of tuples containing the audio file paths and labels\n    data = [(full_img_path, severity) for full_img_path, severity in label_mapping.items()]\n    # Create a Pandas DataFrame from the list of tuples\n    annotated_data = pd.DataFrame(data, columns=['full_img_path', 'severity'])\n\n    label_encoder = LabelEncoder()\n\n    annotated_data['encoded_severity'] = label_encoder.fit_transform(annotated_data['severity'])\n    print(annotated_data.shape)\n    print(annotated_data.head(5))\n    \n    file_name = \"{0}_feature_metadata\".format(condition)\n\n    annotated_data.to_csv(\"{0}.csv\".format(file_name), index=False)\n    print(\"saved file {0}\".format(file_name))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=1000)\npl.Config.set_tbl_rows(1000)\n\ntest = pl.read_csv('/kaggle/input/left-neural-foraminal-narrowing-metadata/left_neural_foraminal_narrowing_l4_l5_feature_metadata.csv'\\\n            , low_memory=True)\n\nprint(test.shape)\nprint(test.head(5))","metadata":{"execution":{"iopub.status.busy":"2024-07-28T15:46:08.321407Z","iopub.execute_input":"2024-07-28T15:46:08.322514Z","iopub.status.idle":"2024-07-28T15:46:08.593283Z","shell.execute_reply.started":"2024-07-28T15:46:08.322463Z","shell.execute_reply":"2024-07-28T15:46:08.592111Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"(147083, 3)\nshape: (5, 3)\n┌────────────────────────────────────────────────────────────────────┬──────────┬──────────────────┐\n│ full_img_path                                                      ┆ severity ┆ encoded_severity │\n│ ---                                                                ┆ ---      ┆ ---              │\n│ str                                                                ┆ str      ┆ i64              │\n╞════════════════════════════════════════════════════════════════════╪══════════╪══════════════════╡\n│ /kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/t ┆ Moderate ┆ 0                │\n│ rain_images/1038453736/3170465859/12.dcm                           ┆          ┆                  │\n│ /kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/t ┆ Moderate ┆ 0                │\n│ rain_images/1038453736/3170465859/18.dcm                           ┆          ┆                  │\n│ /kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/t ┆ Moderate ┆ 0                │\n│ rain_images/1038453736/3170465859/9.dcm                            ┆          ┆                  │\n│ /kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/t ┆ Moderate ┆ 0                │\n│ rain_images/1038453736/3170465859/14.dcm                           ┆          ┆                  │\n│ /kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/t ┆ Moderate ┆ 0                │\n│ rain_images/1038453736/3170465859/11.dcm                           ┆          ┆                  │\n└────────────────────────────────────────────────────────────────────┴──────────┴──────────────────┘\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Checking the output of downsampling","metadata":{}},{"cell_type":"code","source":"dicom_ds = dcmread('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/1038453736/3170465859/12.dcm')\nimg_array = dicom_ds.pixel_array\nIMG_normalized = cv2.normalize(img_array, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The image has {} x {} voxels'.format(img_array.shape[0],img_array.shape[1]))\ndata_downsampling = img_array[::2, ::2]\nprint('The downsampled image has {} x {} voxels'.format(data_downsampling.shape[0], data_downsampling.shape[1]))\n\n# copy the data back to the original data set\ndicom_ds.PixelData = data_downsampling.tobytes()\n# update the information regarding the shape of the data array\ndicom_ds.Rows, dicom_ds.Columns = data_downsampling.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(IMG_normalized)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"left_neural_foraminal_narrowing_l4_l5 - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(data_downsampling)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"left_neural_foraminal_narrowing_l4_l5 - downsampled - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_training_data.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df_w_series_desc = dd.sql(\"select  t.*, ftd.series_description \\\nfrom test t \\\njoin full_training_data ftd \\\non ftd.study_id = cast(split_part(t.full_img_path,'/',-3) as int64) \\\nand ftd.series_id = cast(split_part(t.full_img_path,'/',-2) as int64) \\\nwhere ftd.condition_vert_level = 'left_neural_foraminal_narrowing_l4_l5' and ftd.severity is not null\\\n\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df_w_series_desc.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df = metadata_df_w_series_desc.filter(pl.col('series_description')=='Sagittal T1')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_type_list = dd.sql(\"select distinct series_description from metadata_df_w_series_desc\").pl().to_series().to_list()\nseries_type_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for s_type in series_type_list:\n    metadata_df = metadata_df_w_series_desc.filter(pl.col('series_description')==s_type)\n    print(\"for images of orientation :\", s_type)\n    labels = []\n    features = []\n    for i in tqdm(range(metadata_df.shape[0])):\n        dcm_image_path = metadata_df.item(i,0)\n        dicom_ds = dcmread(dcm_image_path)\n        img_array = dicom_ds.pixel_array\n        #IMG_normalized = cv2.normalize(img_array, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n        features.append(np.mean(img_array.T, axis=0))\n        labels.append(metadata_df.item(i,2))\n    extracted_training_features = pd.DataFrame(features, columns=['image_array'])\n    extracted_training_labels = pd.DataFrame(labels, columns=['encoded_severity'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features[10000].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select distinct severity, encoded_severity from test order by encoded_severity\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### feature extraction from Image files","metadata":{}},{"cell_type":"code","source":"metadata_root_path = '/kaggle/input/left-neural-foraminal-narrowing-metadata'\n\nfor file in os.listdir(metadata_root_path):\n    if file == 'left_neural_foraminal_narrowing_l5_s1_feature_metadata.csv':\n        labels = []\n        features = []\n        data = {}\n        metadata_file_path = os.path.join(metadata_root_path, file)\n        print(metadata_file_path)\n        metadata_df = pl.read_csv(metadata_file_path, low_memory=True)\n        for j in tqdm(range(metadata_df.shape[0])):\n            dcm_image_path = metadata_df.item(j,0)\n            dicom_ds = dcmread(dcm_image_path)\n            img_array = dicom_ds.pixel_array\n            features.append(np.mean(img_array.T, axis=0))\n            # features.append(dcmread(dcm_image_path).pixel_array)\n            labels.append(metadata_df.item(j,2))\n\n        print(\"feature list length --> \", len(features))\n        print(\"label list length --> \", len(labels))\n\n        #extracted_training_features = np.array([np.resize(img,(128,128)) for img in features])\n        #training_labels = np.array(labels)\n\n        #del labels\n        #del features\n        #gc.collect()\n\n        extracted_training_features_file_name = \"{0}_training_features\".format(file.replace('_feature_metadata.csv',''))\n        labels_file_name = \"{0}_labels\".format(file.replace('.csv',''))\n\n        with open(extracted_training_features_file_name, \"wb\") as file:\n            pickle.dump(features, file)\n        with open(labels_file_name, \"wb\") as file:\n            pickle.dump(labels, file)\n\n        del labels\n        del features\n        gc.collect()\n        libc = ctypes.CDLL(\"libc.so.6\") # clearing cache \n        libc.malloc_trim(0)\n\n        #data[\"image_array\"] = features\n        #data[\"encoded_severity\"] = labels\n        #extracted_training_data = pd.DataFrame(data)\n        #extracted_training_data.to_csv(\"{0}.csv.gz\".format(file.replace('.csv','')), index=False, compression='gzip')\n\n        print('finished dumping features & labels for {0}'.format(file))","metadata":{"execution":{"iopub.status.busy":"2024-07-29T05:54:09.211439Z","iopub.execute_input":"2024-07-29T05:54:09.212442Z","iopub.status.idle":"2024-07-29T06:43:16.260597Z","shell.execute_reply.started":"2024-07-29T05:54:09.212406Z","shell.execute_reply":"2024-07-29T06:43:16.258037Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"/kaggle/input/left-neural-foraminal-narrowing-metadata/left_neural_foraminal_narrowing_l5_s1_feature_metadata.csv\n","output_type":"stream"},{"name":"stderr","text":"100%|██████████| 147083/147083 [49:04<00:00, 49.96it/s] \n","output_type":"stream"},{"name":"stdout","text":"feature list length -->  147083\nlabel list length -->  147083\nfinished dumping features & labels for <_io.BufferedWriter name='left_neural_foraminal_narrowing_l5_s1_feature_metadata_labels'>\n","output_type":"stream"}]},{"cell_type":"code","source":"features = []\nlabels = []\n# data = {}\n\nfor j in tqdm(range(test.head(100).shape[0])):\n    dcm_image_path = test.item(j,0)\n    dicom_ds = dcmread(dcm_image_path)\n    img_array = dicom_ds.pixel_array\n    #print(img_array.shape)\n    # features.append(img_array)\n    features.append(np.mean(img_array.T, axis=0))\n    labels.append(test.item(j,2))\n\nprint(\"feature list length --> \", len(features))\nprint(\"label list length --> \", len(labels))\n\nextracted_training_features = features\nlabels = labels\n\n#data[\"image_array\"] = features\n#data[\"encoded_severity\"] = labels\n\n#extracted_training_data = pd.DataFrame(data)\n#extracted_training_data.to_csv(\"test.csv\", index=False)\n\nprint('finished dumping features & labels')","metadata":{"execution":{"iopub.status.busy":"2024-07-28T15:46:52.840483Z","iopub.execute_input":"2024-07-28T15:46:52.841328Z","iopub.status.idle":"2024-07-28T15:46:55.480033Z","shell.execute_reply.started":"2024-07-28T15:46:52.841292Z","shell.execute_reply":"2024-07-28T15:46:55.478975Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"100%|██████████| 100/100 [00:02<00:00, 38.08it/s]","output_type":"stream"},{"name":"stdout","text":"feature list length -->  100\nlabel list length -->  100\nfinished dumping features & labels\n","output_type":"stream"},{"name":"stderr","text":"\n","output_type":"stream"}]},{"cell_type":"code","source":"with open(\"/kaggle/input/left-neural-foraminal-narrowing-training/left_neural_foraminal_narrowing_l1_l2_training_features\", \"rb\") as file:\n    extracted_training_features = pickle.load(file)\n    \nwith open(\"/kaggle/input/left-neural-foraminal-narrowing-training/left_neural_foraminal_narrowing_l1_l2_feature_metadata_labels\", \"rb\") as file:\n    labels = pickle.load(file)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:49:06.727209Z","iopub.execute_input":"2024-08-03T15:49:06.727968Z","iopub.status.idle":"2024-08-03T15:49:11.393178Z","shell.execute_reply.started":"2024-08-03T15:49:06.727935Z","shell.execute_reply":"2024-08-03T15:49:11.392407Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"print(type(extracted_training_features[5]))\nprint(extracted_training_features[5].shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:41:14.405071Z","iopub.execute_input":"2024-08-03T15:41:14.405831Z","iopub.status.idle":"2024-08-03T15:41:14.412492Z","shell.execute_reply.started":"2024-08-03T15:41:14.405791Z","shell.execute_reply":"2024-08-03T15:41:14.411013Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"<class 'numpy.ndarray'>\n(540,)\n","output_type":"stream"}]},{"cell_type":"code","source":"labels_vstacked = np.vstack(labels)\nlabels_vstacked.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:52:54.673915Z","iopub.execute_input":"2024-08-03T15:52:54.674265Z","iopub.status.idle":"2024-08-03T15:52:54.996334Z","shell.execute_reply.started":"2024-08-03T15:52:54.674236Z","shell.execute_reply":"2024-08-03T15:52:54.995397Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"(147083, 1)"},"metadata":{}}]},{"cell_type":"code","source":"extracted_training_features_resized = []\n\nfor img in extracted_training_features:\n    extracted_training_features_resized.append(np.resize(img,(128,)))\n    \nfinal = np.array(extracted_training_features_resized)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:49:18.306307Z","iopub.execute_input":"2024-08-03T15:49:18.307149Z","iopub.status.idle":"2024-08-03T15:49:19.921799Z","shell.execute_reply.started":"2024-08-03T15:49:18.307116Z","shell.execute_reply":"2024-08-03T15:49:19.921023Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"print(type(final))\nprint(final.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:49:26.269476Z","iopub.execute_input":"2024-08-03T15:49:26.269889Z","iopub.status.idle":"2024-08-03T15:49:26.274863Z","shell.execute_reply.started":"2024-08-03T15:49:26.269860Z","shell.execute_reply":"2024-08-03T15:49:26.273887Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"<class 'numpy.ndarray'>\n(147083, 128)\n","output_type":"stream"}]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Flatten(input_shape=[128]))\nmodel.add(keras.layers.Dense(150, activation=\"relu\"))\nmodel.add(keras.layers.Dense(65, activation=\"relu\"))\nmodel.add(keras.layers.Dense(3, activation=\"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:16:21.637684Z","iopub.execute_input":"2024-08-03T16:16:21.638038Z","iopub.status.idle":"2024-08-03T16:16:21.677673Z","shell.execute_reply.started":"2024-08-03T16:16:21.638009Z","shell.execute_reply":"2024-08-03T16:16:21.676838Z"},"trusted":true},"execution_count":21,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/keras/src/layers/reshaping/flatten.py:37: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n  super().__init__(**kwargs)\n","output_type":"stream"}]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train, x_test, y_train, y_test = train_test_split(final, labels_vstacked, test_size=0.4, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:53:07.510354Z","iopub.execute_input":"2024-08-03T15:53:07.511219Z","iopub.status.idle":"2024-08-03T15:53:07.582254Z","shell.execute_reply.started":"2024-08-03T15:53:07.511188Z","shell.execute_reply":"2024-08-03T15:53:07.581466Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\nprint(y_train.shape)\nprint(x_test.shape)\nprint(y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:53:41.535658Z","iopub.execute_input":"2024-08-03T15:53:41.535995Z","iopub.status.idle":"2024-08-03T15:53:41.541243Z","shell.execute_reply.started":"2024-08-03T15:53:41.535971Z","shell.execute_reply":"2024-08-03T15:53:41.540350Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"(88249, 128)\n(88249, 1)\n(58834, 128)\n(58834, 1)\n","output_type":"stream"}]},{"cell_type":"code","source":"x_test, x_valid, y_test, y_valid = train_test_split(x_test, y_test, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:54:30.737507Z","iopub.execute_input":"2024-08-03T15:54:30.738186Z","iopub.status.idle":"2024-08-03T15:54:30.771082Z","shell.execute_reply.started":"2024-08-03T15:54:30.738154Z","shell.execute_reply":"2024-08-03T15:54:30.769988Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"print(x_train.shape)\nprint(y_train.shape)\nprint(x_test.shape)\nprint(y_test.shape)\nprint(x_valid.shape)\nprint(y_valid.shape)","metadata":{"execution":{"iopub.status.busy":"2024-08-03T15:54:53.480282Z","iopub.execute_input":"2024-08-03T15:54:53.480898Z","iopub.status.idle":"2024-08-03T15:54:53.486152Z","shell.execute_reply.started":"2024-08-03T15:54:53.480866Z","shell.execute_reply":"2024-08-03T15:54:53.485141Z"},"trusted":true},"execution_count":14,"outputs":[{"name":"stdout","text":"(88249, 128)\n(88249, 1)\n(47067, 128)\n(47067, 1)\n(11767, 128)\n(11767, 1)\n","output_type":"stream"}]},{"cell_type":"code","source":"import tensorflow.keras.backend as K\nK.epsilon()","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:17:17.653020Z","iopub.execute_input":"2024-08-03T16:17:17.653820Z","iopub.status.idle":"2024-08-03T16:17:17.660792Z","shell.execute_reply.started":"2024-08-03T16:17:17.653783Z","shell.execute_reply":"2024-08-03T16:17:17.659739Z"},"trusted":true},"execution_count":24,"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"1e-07"},"metadata":{}}]},{"cell_type":"code","source":"model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\"\n              , metrics=[keras.metrics.SparseCategoricalCrossentropy()])","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:30:05.914137Z","iopub.execute_input":"2024-08-03T16:30:05.914512Z","iopub.status.idle":"2024-08-03T16:30:05.932245Z","shell.execute_reply.started":"2024-08-03T16:30:05.914472Z","shell.execute_reply":"2024-08-03T16:30:05.931375Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x_train, y_train, epochs=120, validation_data=(x_valid, y_valid))","metadata":{"execution":{"iopub.status.busy":"2024-08-03T16:30:16.974193Z","iopub.execute_input":"2024-08-03T16:30:16.974577Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Epoch 1/120\n\u001b[1m 111/2758\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m3s\u001b[0m 1ms/step - loss: 11.7428 - sparse_categorical_crossentropy: 11.7428  ","output_type":"stream"},{"name":"stderr","text":"WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\nI0000 00:00:1722702619.542056     122 device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n","output_type":"stream"},{"name":"stdout","text":"\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 2ms/step - loss: 2.6047 - sparse_categorical_crossentropy: 2.6047 - val_loss: 0.3744 - val_sparse_categorical_crossentropy: 0.3744\nEpoch 2/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.2107 - sparse_categorical_crossentropy: 0.2107 - val_loss: 0.1619 - val_sparse_categorical_crossentropy: 0.1619\nEpoch 3/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 1ms/step - loss: 0.1590 - sparse_categorical_crossentropy: 0.1590 - val_loss: 0.1621 - val_sparse_categorical_crossentropy: 0.1621\nEpoch 4/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 1ms/step - loss: 0.1614 - sparse_categorical_crossentropy: 0.1614 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 5/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1622 - sparse_categorical_crossentropy: 0.1622 - val_loss: 0.1632 - val_sparse_categorical_crossentropy: 0.1632\nEpoch 6/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1608 - sparse_categorical_crossentropy: 0.1608 - val_loss: 0.1641 - val_sparse_categorical_crossentropy: 0.1641\nEpoch 7/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 1ms/step - loss: 0.1664 - sparse_categorical_crossentropy: 0.1664 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 8/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1612 - sparse_categorical_crossentropy: 0.1612 - val_loss: 0.1632 - val_sparse_categorical_crossentropy: 0.1632\nEpoch 9/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 1ms/step - loss: 0.1601 - sparse_categorical_crossentropy: 0.1601 - val_loss: 0.1630 - val_sparse_categorical_crossentropy: 0.1630\nEpoch 10/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 1ms/step - loss: 0.1605 - sparse_categorical_crossentropy: 0.1605 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 11/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 1ms/step - loss: 0.1630 - sparse_categorical_crossentropy: 0.1630 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 12/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1672 - sparse_categorical_crossentropy: 0.1672 - val_loss: 0.1630 - val_sparse_categorical_crossentropy: 0.1630\nEpoch 13/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1629 - sparse_categorical_crossentropy: 0.1629 - val_loss: 0.1631 - val_sparse_categorical_crossentropy: 0.1631\nEpoch 14/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1614 - sparse_categorical_crossentropy: 0.1614 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 15/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1615 - sparse_categorical_crossentropy: 0.1615 - val_loss: 0.1627 - val_sparse_categorical_crossentropy: 0.1627\nEpoch 16/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1646 - sparse_categorical_crossentropy: 0.1646 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 17/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1613 - sparse_categorical_crossentropy: 0.1613 - val_loss: 0.1630 - val_sparse_categorical_crossentropy: 0.1630\nEpoch 18/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1663 - sparse_categorical_crossentropy: 0.1663 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 19/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1643 - sparse_categorical_crossentropy: 0.1643 - val_loss: 0.1643 - val_sparse_categorical_crossentropy: 0.1643\nEpoch 20/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1603 - sparse_categorical_crossentropy: 0.1603 - val_loss: 0.1632 - val_sparse_categorical_crossentropy: 0.1632\nEpoch 21/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1644 - sparse_categorical_crossentropy: 0.1644 - val_loss: 0.1635 - val_sparse_categorical_crossentropy: 0.1635\nEpoch 22/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1642 - sparse_categorical_crossentropy: 0.1642 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 23/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1618 - sparse_categorical_crossentropy: 0.1618 - val_loss: 0.1630 - val_sparse_categorical_crossentropy: 0.1630\nEpoch 24/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1619 - sparse_categorical_crossentropy: 0.1619 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 25/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1627 - sparse_categorical_crossentropy: 0.1627 - val_loss: 0.1631 - val_sparse_categorical_crossentropy: 0.1631\nEpoch 26/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1600 - sparse_categorical_crossentropy: 0.1600 - val_loss: 0.1631 - val_sparse_categorical_crossentropy: 0.1631\nEpoch 27/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1629 - sparse_categorical_crossentropy: 0.1629 - val_loss: 0.1649 - val_sparse_categorical_crossentropy: 0.1649\nEpoch 28/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1636 - sparse_categorical_crossentropy: 0.1636 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 29/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1615 - sparse_categorical_crossentropy: 0.1615 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 30/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 2ms/step - loss: 0.1656 - sparse_categorical_crossentropy: 0.1656 - val_loss: 0.1627 - val_sparse_categorical_crossentropy: 0.1627\nEpoch 31/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 2ms/step - loss: 0.1598 - sparse_categorical_crossentropy: 0.1598 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 32/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1635 - sparse_categorical_crossentropy: 0.1635 - val_loss: 0.1631 - val_sparse_categorical_crossentropy: 0.1631\nEpoch 33/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1617 - sparse_categorical_crossentropy: 0.1617 - val_loss: 0.1634 - val_sparse_categorical_crossentropy: 0.1634\nEpoch 34/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1621 - sparse_categorical_crossentropy: 0.1621 - val_loss: 0.1643 - val_sparse_categorical_crossentropy: 0.1643\nEpoch 35/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1597 - sparse_categorical_crossentropy: 0.1597 - val_loss: 0.1633 - val_sparse_categorical_crossentropy: 0.1633\nEpoch 36/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1590 - sparse_categorical_crossentropy: 0.1590 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 37/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1641 - sparse_categorical_crossentropy: 0.1641 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 38/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1618 - sparse_categorical_crossentropy: 0.1618 - val_loss: 0.1631 - val_sparse_categorical_crossentropy: 0.1631\nEpoch 39/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1566 - sparse_categorical_crossentropy: 0.1566 - val_loss: 0.1630 - val_sparse_categorical_crossentropy: 0.1630\nEpoch 40/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1611 - sparse_categorical_crossentropy: 0.1611 - val_loss: 0.1627 - val_sparse_categorical_crossentropy: 0.1627\nEpoch 41/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1600 - sparse_categorical_crossentropy: 0.1600 - val_loss: 0.1629 - val_sparse_categorical_crossentropy: 0.1629\nEpoch 42/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1607 - sparse_categorical_crossentropy: 0.1607 - val_loss: 0.1626 - val_sparse_categorical_crossentropy: 0.1626\nEpoch 43/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1611 - sparse_categorical_crossentropy: 0.1611 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 44/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 2ms/step - loss: 0.1624 - sparse_categorical_crossentropy: 0.1624 - val_loss: 0.1627 - val_sparse_categorical_crossentropy: 0.1627\nEpoch 45/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1656 - sparse_categorical_crossentropy: 0.1656 - val_loss: 0.1628 - val_sparse_categorical_crossentropy: 0.1628\nEpoch 46/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1599 - sparse_categorical_crossentropy: 0.1599 - val_loss: 0.1627 - val_sparse_categorical_crossentropy: 0.1627\nEpoch 47/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1636 - sparse_categorical_crossentropy: 0.1636 - val_loss: 0.1627 - val_sparse_categorical_crossentropy: 0.1627\nEpoch 48/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1618 - sparse_categorical_crossentropy: 0.1618 - val_loss: 0.1626 - val_sparse_categorical_crossentropy: 0.1626\nEpoch 49/120\n\u001b[1m2758/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 2ms/step - loss: 0.1579 - sparse_categorical_crossentropy: 0.1579 - val_loss: 0.1627 - val_sparse_categorical_crossentropy: 0.1627\nEpoch 50/120\n\u001b[1m2743/2758\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.1607 - sparse_categorical_crossentropy: 0.1607","output_type":"stream"}]},{"cell_type":"code","source":"model.evaluate(x_test, y_test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/training-files-lnfn/left_neural_foraminal_narrowing_l1_l2_feature_metadata.csv')\nprint(df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-28T14:56:31.562339Z","iopub.execute_input":"2024-07-28T14:56:31.562740Z","iopub.status.idle":"2024-07-28T14:56:49.723641Z","shell.execute_reply.started":"2024-07-28T14:56:31.562706Z","shell.execute_reply":"2024-07-28T14:56:49.721681Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"(147083, 2)\n","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)","Cell \u001b[0;32mIn[3], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m df \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mread_csv(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/kaggle/input/training-files-lnfn/left_neural_foraminal_narrowing_l1_l2_feature_metadata.csv\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28mprint\u001b[39m(df\u001b[38;5;241m.\u001b[39mshape)\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[43mdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtypes\u001b[49m)\n","\u001b[0;31mAttributeError\u001b[0m: 'Index' object has no attribute 'types'"],"ename":"AttributeError","evalue":"'Index' object has no attribute 'types'","output_type":"error"}]},{"cell_type":"code","source":"df['image_array'][0]","metadata":{"execution":{"iopub.status.busy":"2024-07-28T14:58:23.742394Z","iopub.execute_input":"2024-07-28T14:58:23.742791Z","iopub.status.idle":"2024-07-28T14:58:23.751565Z","shell.execute_reply.started":"2024-07-28T14:58:23.742758Z","shell.execute_reply":"2024-07-28T14:58:23.750161Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":8,"outputs":[{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"'[  0.203125    58.95833333  63.41927083  62.55208333  61.95572917\\n  63.0234375   66.19270833  68.76822917  70.76302083  72.94791667\\n  74.33072917  75.42447917  76.29947917  76.45833333  76.37760417\\n  76.08333333  75.79427083  75.62760417  75.625       75.52604167\\n  75.36979167  75.61197917  76.09895833  76.97395833  78.1328125\\n  79.01041667  79.1796875   78.546875    77.5546875   77.1328125\\n  77.28385417  77.83072917  78.81510417  79.75260417  80.4765625\\n  80.8515625   80.99739583  80.87239583  80.29427083  79.51302083\\n  77.70052083  75.65885417  74.3359375   73.53125     73.90364583\\n  75.53385417  78.1796875   80.6328125   82.55729167  83.40364583\\n  83.625       83.66145833  83.84375     83.96614583  83.88802083\\n  83.59895833  83.2890625   83.24739583  83.47135417  83.78645833\\n  84.47395833  85.4765625   86.20833333  86.625       86.8046875\\n  86.94010417  86.671875    86.46614583  86.75260417  87.54947917\\n  88.45052083  89.02864583  88.97916667  88.34114583  87.64322917\\n  87.08854167  87.12239583  87.96875     88.77083333  88.94791667\\n  88.2578125   87.04947917  86.11197917  86.03645833  87.0546875\\n  88.44010417  90.04166667  91.92708333  93.89583333  95.53385417\\n  96.40364583  96.08333333  95.21614583  94.78125     94.765625\\n  94.75260417  94.56770833  93.8515625   91.65104167  88.43229167\\n  85.76822917  84.81510417  86.859375    90.50260417  93.48958333\\n  94.94010417  95.7109375   95.92447917  96.47916667  96.92708333\\n  97.24479167  97.3046875   97.484375    97.54166667  97.51302083\\n  97.36979167  96.84375     96.07291667  95.51041667  95.59895833\\n  96.02604167  96.58854167  97.14322917  97.03125     95.90364583\\n  94.19791667  92.84375     91.95572917  91.68489583  91.84114583\\n  92.57291667  93.7890625   94.85416667  95.66145833  96.43229167\\n  96.88541667  97.33854167  97.5625      97.40104167  97.5625\\n  97.54166667  97.01041667  96.21614583  95.41145833  94.8203125\\n  94.55208333  94.90625     95.796875    97.05208333  97.75\\n  97.8125      97.34375     97.72135417  99.38020833 101.42447917\\n 102.1328125  101.88541667 100.3046875   98.328125    95.92447917\\n  93.72395833  91.64322917  90.33333333  90.81770833  93.38541667\\n  95.97916667  97.33854167  96.9609375   95.26302083  93.6015625\\n  92.40625     92.203125    92.41666667  92.4375      92.23697917\\n  91.97916667  91.7265625   91.05729167  89.72916667  88.83854167\\n  87.97916667  87.85416667  87.9765625   87.40364583  87.19270833\\n  87.75520833  88.96614583  90.421875    92.078125    93.03125\\n  92.078125    90.0078125   87.56770833  85.07552083  81.5703125\\n  78.99739583  76.875       76.53125     76.41666667  75.8671875\\n  73.71614583  71.01302083  69.1171875   68.80729167  68.68229167\\n  68.640625    68.52604167  67.83072917  66.3984375   65.43229167\\n  64.26822917  64.16145833  63.90625     64.40364583  64.43229167\\n  64.12760417  63.69010417  63.7421875   64.66927083  64.7265625\\n  64.19791667  64.6875      63.40885417  61.          57.8125\\n  55.9609375   56.4375      58.22395833  60.328125    61.5\\n  61.95572917  61.328125    60.55989583  60.42447917  60.7578125\\n  61.73697917  63.55989583  64.953125    65.33072917  64.61197917\\n  63.17447917  61.85416667  60.09375     58.77083333  57.05729167\\n  55.10677083  53.09895833  51.97916667  51.88802083  52.29947917\\n  52.55729167  53.03125     53.55989583  54.0546875   55.19791667\\n  56.63020833  57.05729167  56.69010417  55.609375    54.48697917\\n  54.00520833  53.75        53.96875     54.40104167  54.09375\\n  53.7265625   53.5         52.90364583  51.40104167  49.703125\\n  48.30989583  48.49739583  50.6796875   53.90104167  56.54166667\\n  56.55989583  55.10416667  54.96875     57.27864583  60.29166667\\n  63.27083333  64.59635417  63.6953125   61.54166667  60.765625\\n  61.921875    63.69791667  65.265625    66.44791667  67.09635417\\n  68.81770833  72.76822917  76.04947917  77.56770833  78.52864583\\n  78.99479167  78.90625     77.3203125   74.63541667  70.36458333\\n  66.8125      65.55729167  65.77604167  66.60677083  67.41145833\\n  67.63541667  67.32552083  67.1171875   67.61458333  67.09635417\\n  67.4140625   68.53125     68.63020833  67.51041667  65.05208333\\n  63.1875      61.73697917  61.65364583  62.32552083  62.7890625\\n  61.87760417  60.3046875   58.9765625   58.17447917  58.11458333\\n  57.96875     57.95052083  59.58333333  62.6640625   65.9453125\\n  70.          72.55729167  72.55208333  72.88020833  74.03645833\\n  75.890625    79.26822917  83.21354167  89.81510417  98.13541667\\n 105.66927083 108.7421875  107.56770833 104.46614583 101.57552083\\n  99.94010417 100.21354167 101.25       102.015625   102.015625\\n 102.91666667 104.48958333 107.00260417 109.8046875  112.66927083\\n 114.7890625  115.97916667 117.39322917 121.         124.54166667\\n 127.70572917 128.27083333 127.07552083 125.79427083 124.265625\\n 123.79947917 123.9296875  124.60416667 125.8359375  125.88020833\\n 123.390625   119.93229167 117.18489583 116.55208333 116.95572917\\n 117.32291667 115.09375    111.2578125  108.765625   109.29166667\\n 111.60677083 115.92708333 120.75       124.09635417 126.05208333\\n 124.8984375  124.453125   123.82552083 121.984375   121.33854167\\n 122.49479167 123.19270833 123.3359375  122.3515625  120.3359375\\n 118.25260417 115.85416667 114.47395833 113.8984375  112.9765625\\n 112.96875    113.78645833 114.3515625  113.95052083 112.2421875\\n 110.71875    108.15625    105.96614583 103.86197917 102.27864583\\n  99.98697917  97.96354167  95.8359375   93.66927083  91.04166667\\n  89.17708333  86.20052083  83.83333333  80.69010417  77.85416667\\n  74.3984375   70.97395833  67.86979167  65.16927083  63.36979167\\n  63.234375    64.68489583  65.84895833  65.68229167  65.13020833\\n  64.60416667  64.66927083  65.59114583  66.875       68.12239583\\n  69.13802083  70.42708333  71.44010417  71.87760417  71.73958333\\n  71.40885417  71.1875      71.546875    72.2890625   72.75\\n  72.85677083  72.87760417  73.078125    74.74479167  76.12239583\\n  77.859375    78.78385417  79.34114583  78.7109375   77.33072917\\n  76.38541667  76.13802083  75.95052083  75.90625     74.90885417\\n  74.42708333  74.54427083  74.4140625   74.20052083  74.2421875\\n  73.95052083  72.72135417  71.27604167  70.4453125   70.5859375\\n  71.68489583  72.734375    73.45572917  73.84895833  73.78125\\n  73.28645833  72.4453125   71.375       69.8671875   68.25260417\\n  67.17708333  67.8203125   69.27083333  70.09114583  70.02864583\\n  68.72916667  67.48697917  67.39322917  68.28125     69.79427083\\n  71.23177083  71.7421875   70.86458333  69.48177083  68.7890625\\n  69.2265625   69.66927083  69.80208333  69.55989583  69.36979167\\n  70.61197917  72.96354167  75.77864583  78.31510417  80.70833333\\n  82.80208333  85.50520833  88.21354167  90.51822917  92.29947917\\n  93.39583333  94.36197917  94.8125      95.37239583  95.53385417\\n  95.0703125   94.72916667  94.4765625   94.42447917  94.72916667\\n  95.96354167  96.92708333  98.02604167  99.01822917 100.0234375\\n 101.38802083 103.95833333 107.0234375  112.19270833 116.00260417\\n 120.0234375  122.01822917 115.83333333 114.60416667 102.12239583\\n  98.07291667  92.49739583  96.8984375   92.70572917   0.90625   ]'"},"metadata":{}}]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nprint(tf.__version__)\nprint(keras.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-07-28T04:25:01.828628Z","iopub.execute_input":"2024-07-28T04:25:01.829022Z","iopub.status.idle":"2024-07-28T04:25:16.649429Z","shell.execute_reply.started":"2024-07-28T04:25:01.828991Z","shell.execute_reply":"2024-07-28T04:25:16.648049Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stderr","text":"2024-07-28 04:25:04.161504: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n2024-07-28 04:25:04.161652: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n2024-07-28 04:25:04.341706: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n","output_type":"stream"},{"name":"stdout","text":"2.15.0\n3.3.3\n","output_type":"stream"}]},{"cell_type":"code","source":"fashion_mnist = keras.datasets.fashion_mnist\n(X_train_full, y_train_full),(X_test,y_test) = fashion_mnist.load_data()","metadata":{"execution":{"iopub.status.busy":"2024-07-28T04:25:20.527248Z","iopub.execute_input":"2024-07-28T04:25:20.527986Z","iopub.status.idle":"2024-07-28T04:25:21.777675Z","shell.execute_reply.started":"2024-07-28T04:25:20.527950Z","shell.execute_reply":"2024-07-28T04:25:21.776127Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n\u001b[1m29515/29515\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 0us/step\nDownloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n\u001b[1m26421880/26421880\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 0us/step\nDownloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n\u001b[1m5148/5148\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1us/step\nDownloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n\u001b[1m4422102/4422102\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 0us/step\n","output_type":"stream"}]},{"cell_type":"code","source":"print(type(X_train_full[0]))\nprint(X_train_full[0].shape)\nprint(X_train_full.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-28T04:26:04.195892Z","iopub.execute_input":"2024-07-28T04:26:04.196338Z","iopub.status.idle":"2024-07-28T04:26:04.202671Z","shell.execute_reply.started":"2024-07-28T04:26:04.196301Z","shell.execute_reply":"2024-07-28T04:26:04.201301Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"<class 'numpy.ndarray'>\n(28, 28)\n(60000, 28, 28)\n","output_type":"stream"}]},{"cell_type":"code","source":"type(y_train_full[0])","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:44:38.320591Z","iopub.execute_input":"2024-07-21T05:44:38.321047Z","iopub.status.idle":"2024-07-21T05:44:38.328087Z","shell.execute_reply.started":"2024-07-21T05:44:38.321013Z","shell.execute_reply":"2024-07-21T05:44:38.326825Z"},"trusted":true},"execution_count":32,"outputs":[{"execution_count":32,"output_type":"execute_result","data":{"text/plain":"numpy.uint8"},"metadata":{}}]},{"cell_type":"code","source":"# from sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\n\nx = test.select(pl.col('full_img_path'))\ny = test.select(pl.col('encoded_severity'))\n\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=42)\n\nprint(\"full train shape & label shape = {0} & {1}\".format(x_train.shape, y_train.shape))\nprint(\"full test shape & label shape = {0} & {1}\".format(x_test.shape, y_test.shape))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select condition_vert_level, series_description, severity, count(distinct(study_id)) as study_count \\\n, count(distinct(series_id)) as series_count from full_training_data \\\ngroup by condition_vert_level, series_description, severity order by 1,2,3\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Looking at the number of studies and series of images available for various scan orientations, it appears that the Saggital orientations are missing for a couple of studies","metadata":{}},{"cell_type":"code","source":"dd.sql(\"select series_description, count(distinct(study_id)) as studies, count(distinct(series_id)) as series \\\nfrom train_series_desc group by series_description\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Checking whether all the different conditions for all the 5 vertebrae are available in all the studies.\n#### It appears that some of the condition/vertebrae combinations are not available in some of the studies\n##### e.g. The images with orientation \"Axial T2\" for the condition \"Right Subarticular Stenosis\" at the vertebra \"L1/L2\" is available in 1812 studies out of a total of 1975 studies","metadata":{}},{"cell_type":"code","source":"part_1 = list(os.listdir('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images'))\n# part_1 = list(filter(lambda x: x.find('.DS') == -1, part_1))\nlen(part_1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj = {{ \n    'study_id': x,\n    'folder_path': f\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/{x}\",\n    'SeriesInstanceUIDs': [] \n}\n    for x in part_1 \n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"p1 = [(x, f\"/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/{x}\") for x in part_1]\nmeta_obj = { p[0]: { 'folder_path': p[1], \n                    'SeriesInstanceUIDs': [] \n                   } \n            for p in p1 }\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for m in meta_obj:\n    meta_obj[m]['SeriesInstanceUIDs'] = list(os.listdir(meta_obj[m]['folder_path']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_desc.filter((pl.col('study_id') == int('100206310')) & (pl.col('series_id') == int('1792451510')))\\\n.select(pl.col('series_description')).item(0,0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# grabs the correspoding series descriptions\nfor k in tqdm(meta_obj):\n    for s in meta_obj[k]['SeriesInstanceUIDs']:\n        if 'SeriesDescriptions' not in meta_obj[k]:\n            meta_obj[k]['SeriesDescriptions'] = []\n        try:\n            meta_obj[k]['SeriesDescriptions'].append(train_series_desc.filter((pl.col('study_id') == int(k)) \\\n                                                                              & (pl.col('series_id') == int(s)))\\\n                                                     .select(pl.col('series_description')).item(0,0))\n        except:\n            print(\"Failed on\", s, k)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ptobj = meta_obj[list(meta_obj.keys())[10]]\nptobj = meta_obj['100206310']\nptobj","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj['image_files'] = []\n\nfor idx, i in enumerate(ptobj['SeriesInstanceUIDs']):\n    print(idx)\n    print(i)\n    print(\"**********\")\n    ptobj['image_files'].append(glob.glob(f\"{ptobj['folder_path']}/{ptobj['SeriesInstanceUIDs'][idx]}/*.dcm\"))\n    \n#len(images)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj['SeriesDescriptions'][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj['image_files'][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ptobj","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in tqdm(meta_obj):\n    meta_obj[k]['image_files'] = []\n    for idx, i in enumerate(meta_obj[k]['SeriesInstanceUIDs']):\n        meta_obj[k]['image_files']\\\n        .append(len(glob.glob(f\"{meta_obj[k]['folder_path']}/{meta_obj[k]['SeriesInstanceUIDs'][idx]}/*.dcm\")))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj['100206310']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_obj[list(meta_obj.keys())[1]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pl.Config(fmt_str_lengths=100)\npl.Config.set_tbl_rows(1000)\ndd.sql(\" \\\nselect tsd.series_description, tlc.condition, tlc.level, \\\ncount(distinct(tlc.study_id)) as studies, count(distinct(tlc.instance_number)) as label_defining_images, \\\nround(count(distinct(tlc.study_id))/1975, 5) as pct_of_total_studies, \\\nfrom train_series_desc tsd \\\njoin train_label_coordinates tlc \\\non tsd.study_id = tlc.study_id \\\nand tsd.series_id = tlc.series_id \\\ngroup by tsd.series_description, tlc.condition, tlc.level \\\norder by 1,2, 4 desc \\\n\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Now let's check how many studies have moderate or severe levels of the condition \"Right Subarticular Stenosis\" at the vertebra \"L1/L2\"","metadata":{}},{"cell_type":"code","source":"dd.sql(\"select value, count(distinct(study_id)) from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' \\\ngroup by value\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select value, count(distinct(study_id)) from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' \\\nand study_id in (select distinct(study_id) from train_label_coordinates where condition = 'Right Subarticular Stenosis'\\\nand level = 'L1/L2') group by value\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_coordinates_list = dd.sql(\"select distinct study_id from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' \\\nand value = 'Normal/Mild' \\\nand study_id in (select distinct(study_id) from train_label_coordinates where condition = 'Right Subarticular Stenosis'\\\nand level = 'L1/L2')\").pl().to_series().unique().to_list()\n\ntrain_meta_data_list = dd.sql(\"select distinct study_id from train_meta_data_melted where variable = 'right_subarticular_stenosis_l1_l2' and value = 'Normal/Mild' \")\\\n.pl().to_series().unique().to_list()\n\nset(train_label_coordinates_list) ^ set(train_meta_data_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The above analysis shows that there are two studies related to Right Subarticular Stenosis of L1/L2 which are not present in the train_label_coordinates file","metadata":{}},{"cell_type":"code","source":"dd.sql(\"select * from train_label_coordinates where study_id in ('3008676218','3303545110')\").pl()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dd.sql(\"select * from train_meta_data where study_id in ('3008676218','3303545110')\").pl().transpose(include_header=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### It seems from above, that the study 3303545110 is only present for Right Subarticular Stenosis of L5/S1 and not of L1/L2 and the study 3008676218 is not present at all for any conditions/vertebrae combination\n#### The questions is then how is 3303545110 labelled as Normal/Mild for Right Subarticular Stenosis of L1/L2 and 3008676218 labelled for all the conditions/vertebrae combination","metadata":{}},{"cell_type":"markdown","source":"#### Below are some of the images from the study_id/patient 100206310 ","metadata":{}},{"cell_type":"code","source":"dcm_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/20.dcm'\nleft_centre_coords = (int(180.355677), int(165.0342))\nright_centre_coords = (int(145.120536), int(162.285714))\nradius = 10\ncolor = (255, 0, 0)  # Red color in BGR\nthickness = 2\n\ndicom_ds = dcmread(dcm_image_path)\nIMG = dicom_ds.pixel_array\n\nIMG_normalized = cv2.normalize(IMG, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n\nleft_IMG_with_circle = cv2.circle(IMG_normalized.copy(), left_centre_coords, radius, color, thickness)\nleft_IMG_with_circle = cv2.cvtColor(left_IMG_with_circle, cv2.COLOR_BGR2RGB)\n\nright_IMG_with_circle = cv2.circle(IMG_normalized.copy(), right_centre_coords, radius, color, thickness)\nright_IMG_with_circle = cv2.cvtColor(right_IMG_with_circle, cv2.COLOR_BGR2RGB)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(left_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Left subarticular stenosis L1/L2 - Normal/Mild\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(right_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Right subarticular stenosis L1/L2 - Normal/Mild\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dcm_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/1012284084/46.dcm'\nleft_centre_coords = (int(168.536252), int(156.27907))\nright_centre_coords = (int(146.263393), int(160.0))\nradius = 10\ncolor = (255, 0, 0)  # Red color in BGR\nthickness = 2\n\ndicom_ds = dcmread(dcm_image_path)\nIMG = dicom_ds.pixel_array\n\nIMG_normalized = cv2.normalize(IMG, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n\nleft_IMG_with_circle = cv2.circle(IMG_normalized.copy(), left_centre_coords, radius, color, thickness)\nleft_IMG_with_circle = cv2.cvtColor(left_IMG_with_circle, cv2.COLOR_BGR2RGB)\n\nright_IMG_with_circle = cv2.circle(IMG_normalized.copy(), right_centre_coords, radius, color, thickness)\nright_IMG_with_circle = cv2.cvtColor(right_IMG_with_circle, cv2.COLOR_BGR2RGB)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(left_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Left subarticular stenosis L4/L5 - Severe\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(right_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Right subarticular stenosis L4/L5 - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dcm_image_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/train_images/100206310/2092806862/4.dcm'\nright_centre_coords = (int(253.393075), int(278.680244))\nradius = 10\ncolor = (255, 0, 0)  # Red color in BGR\nthickness = 2\n\ndicom_ds = dcmread(dcm_image_path)\nIMG = dicom_ds.pixel_array\n\nIMG_normalized = cv2.normalize(IMG, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8U)\n\nright_IMG_with_circle = cv2.circle(IMG_normalized.copy(), right_centre_coords, radius, color, thickness)\nright_IMG_with_circle = cv2.cvtColor(right_IMG_with_circle, cv2.COLOR_BGR2RGB)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(right_IMG_with_circle)\nplt.axis('off')  # Turn off axis numbers and ticks\nplt.title(\"Right neural foraminal narrowing L4/L5 - Moderate\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aim : To select image files to create Train and Validation set\n1. 25% of data will be set aside permanently as a validation set for all experiments. This set won't be used for training/testing/cross-validation\n2. 75% of data will be used for all experiments\n3. Images are 320 * 320 arrays\n### Modelling strategy\n1. Build 25 different models, one per condition/vertebrae combination","metadata":{}},{"cell_type":"markdown","source":"#### Pipeline to make a sample submission to test the submission data creation logic","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/sample_submission.csv')\nsubmission['row_id'] = 'samples'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os\n\nconfig = {}\nconfig['root_file_path'] = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n\nstudies = os.listdir(config['root_file_path'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"studies","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = {}\nweight_dict = {'normal_mild':1, 'moderate':2, 'severe':4}\nconditions = ['spinal_canal_stenosis', 'neural_foraminal_narrowing', 'subarticular_stenosis']\nsides = ['left', 'right']\nvertebrae_levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\nseverity_levels = ['normal_mild', 'moderate', 'severe']\n\nfor c in conditions:\n    for v in vertebrae_levels:\n        if c != 'spinal_canal_stenosis':\n            for s in sides:\n                for st in studies:\n                    #print(st+'_'+s+'_'+c+'_'+v)\n                    rows[st+'_'+s+'_'+c+'_'+v] = np.array([0.333333 * 2, 0.333333 * 2, 0.333333 * 2])\n        else:\n            for st in studies:\n                #print(st+'_'+c+'_'+v)\n                rows[st+'_'+c+'_'+v] = np.array([0.333333 * 2, 0.333333 * 2, 0.333333 * 2])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for row_id, feature in tqdm(rows.items()):\n    feature_set_reshaped = feature.reshape(1, -1)\n    predictions = np.ascontiguousarray(feature_set_reshaped/2)\n    df = pd.DataFrame(predictions, columns=severity_levels)\n    df.insert(loc=0, column='row_id', value=row_id)\n    submission = pd.concat([submission,df]).reset_index(drop=True)\n    \ni = submission[(submission.row_id == 'samples')].index\nsubmission = submission.drop(i).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}