embedding_generation_data_tables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kyle/supernova/snfactory/manifold_twins/analysis/twins_embedding.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "from idrtools import math\n",
    "from twins_embedding import TwinsEmbeddingAnalysis\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib import colors\n",
    "from tqdm.notebook import tqdm\n",
    "from astropy.table import Table\n",
    "import utils\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n",
      "    IDR:          BLACKSTON\n",
      "    Phase range: [-5.0, 5.0] days\n",
      "    Bin velocity: 1000.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "610f393f40d041d2924a7da6e71952be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/415 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating the spectra at maximum light...\n",
      "    Loaded cached stan model\n",
      "    Using saved stan result\n",
      "Reading between the lines...\n",
      "    Loaded cached stan model\n",
      "    Using saved stan result\n",
      "Building masks...\n",
      "    Masking 30/203 targets whose uncertainty power is \n",
      "    more than 0.100 of the intrinsic power.\n",
      "Generating the manifold learning embedding...\n",
      "Loading other indicators of diversity...\n",
      "Fitting RBTL Twins Manifold GP...\n",
      "GP magnitude residuals fit:\n",
      "    Fit result:           CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH\n",
      "    intrinsic_dispersion      0.073 ± 0.008\n",
      "    gp_kernel_amplitude       0.164 ± 0.082\n",
      "    gp_length_scale           5.541 ± 3.315\n",
      "    offset                    -0.116 ± 0.124\n",
      "    covariate_slope_0         -0.122 ± 0.051\n",
      "    Fit NMAD                  0.083 mag\n",
      "    Fit std                   0.101 mag\n",
      "Fitting SALT2 Twins Manifold GP...\n",
      "GP magnitude residuals fit:\n",
      "    Fit result:           CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH\n",
      "    intrinsic_dispersion      0.085 ± 0.010\n",
      "    gp_kernel_amplitude       0.380 ± 0.230\n",
      "    gp_length_scale           8.631 ± 5.328\n",
      "    offset                    6.956 ± 0.299\n",
      "    covariate_slope_0         2.810 ± 0.153\n",
      "    Fit NMAD                  0.105 mag\n",
      "    Fit std                   0.118 mag\n",
      "Calculating SALT2 magnitude residuals...\n",
      "SALT2 magnitude residuals fit: \n",
      "    alpha:   0.148\n",
      "    beta:    2.706\n",
      "    σ_int:   0.118\n",
      "    RMS:     0.140\n",
      "    NMAD:    0.106\n",
      "    WRMS:    0.140\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "a = TwinsEmbeddingAnalysis()\n",
    "a.settings['figure_directory'] = './output_generation/figures/'\n",
    "a.settings['latex_directory'] = './output_generation/latex/'\n",
    "a.run_analysis()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Attrition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\newcommand{\\numdatasetsne}{280}\n",
      "\\newcommand{\\numdatasetspectra}{3908}\n",
      "\n",
      "\\newcommand{\\nummanifoldsne}{203}\n",
      "\\newcommand{\\nummanifoldspectra}{598}\n",
      "\\newcommand{\\numinterpsne}{173}\n"
     ]
    }
   ],
   "source": [
    "with a.latex_open('attrition_variables.tex') as f:\n",
    "    utils.latex_command(f, 'numdatasetsne', '%d', a.attrition_enough_spectra)\n",
    "    utils.latex_command(f, 'numdatasetspectra', '%d', a.attrition_total_spectra)\n",
    "    utils.latex_print(f, \"\")\n",
    "    utils.latex_command(f, 'nummanifoldsne', '%d', len(a.targets))\n",
    "    utils.latex_command(f, 'nummanifoldspectra', '%d', len(a.spectra))\n",
    "    utils.latex_command(f, 'numinterpsne', '%d', np.sum(a.uncertainty_mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\textbf{General selection requirements} & \\\\\n",
      "Initial sample (SNe Ia with at least 5 SNfactory spectra) & 280 \\\\\n",
      "SALT2 date of maximum light uncertainty < 1 day           & 272 \\\\\n",
      "At least one spectrum within 5 days of maximum light      & 250 \\\\\n",
      "At least one spectrum with S/N 3300-3800~\\AA\\ > 100       & 203 \\\\\n",
      "\\hline\n",
      "\\textbf{Manifold learning selection requirements} & \\\\\n",
      "\\textbf{(Section~\\ref{sec:isomap_sample})} & \\\\\n",
      "Spectrum at max. uncertainty < 10\\% of intrinsic variance        & 173 \\\\\n"
     ]
    }
   ],
   "source": [
    "with a.latex_open('attrition_table.tex') as f:\n",
    "    utils.latex_print(f, \"\\\\textbf{General selection requirements} & \\\\\\\\\")\n",
    "    utils.latex_print(f, \"Initial sample (SNe Ia with at least 5 SNfactory spectra) & %d \\\\\\\\\" % a.attrition_enough_spectra)\n",
    "    utils.latex_print(f, \"SALT2 date of maximum light uncertainty < 1 day           & %d \\\\\\\\\" % a.attrition_salt_daymax)\n",
    "    utils.latex_print(f, \"At least one spectrum within 5 days of maximum light      & %d \\\\\\\\\" % a.attrition_range)\n",
    "    utils.latex_print(f, \"At least one spectrum with S/N 3300-3800~\\AA\\ > 100       & %d \\\\\\\\\" % a.attrition_usable)\n",
    "    utils.latex_print(f, \"\\hline\")\n",
    "    utils.latex_print(f, \"\\\\textbf{Manifold learning selection requirements} & \\\\\\\\\")\n",
    "    utils.latex_print(f, r\"\\textbf{(Section~\\ref{sec:isomap_sample})} & \\\\\")\n",
    "    utils.latex_print(f, \"Spectrum at max. uncertainty < 10\\\\%% of intrinsic variance        & %d \\\\\\\\\" % np.sum(a.uncertainty_mask))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dump of the embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rbtl_mag_errs, rbtl_color_errs = a._calculate_rbtl_uncertainties()\n",
    "pv_errs = a.calculate_peculiar_velocity_uncertainties(a.redshifts)\n",
    "mag_errs = np.sqrt(rbtl_mag_errs**2 + pv_errs**2)\n",
    "\n",
    "data_table = Table({\n",
    "    'name': [a.iau_name_map.get(i.name, i.name).replace('_', '\\_') for i in a.targets],\n",
    "    # 'SALT2 $x_1$': ['%.2f $\\pm$ %.2f' % (i, j) for i, j in zip(a.salt_x1, a.salt_fits['x1_err'])],\n",
    "    # 'SALT2 $color$': ['%.2f $\\pm$ %.2f' % (i, j) for i, j in zip(a.salt_colors, a.salt_fits['c_err'])],\n",
    "    'SALT2 $x_1$': ['%.2f' % i for i in a.salt_x1],\n",
    "    'SALT2 $x_1$ uncertainty': ['%.2f' % i for i in a.salt_fits['x1_err']],\n",
    "    'SALT2 $c$': ['%.3f' % i for i in a.salt_colors],\n",
    "    'SALT2 $c$ uncertainty': ['%.3f' % i for i in a.salt_fits['c_err']],\n",
    "    'RBTL $A_V$': ['%.3f' % i for i in a.rbtl_colors],\n",
    "    'RBTL $A_V$ uncertainty': ['%.3f' % i for i in rbtl_color_errs],\n",
    "    'RBTL $\\Delta M$': ['%.3f' % i for i in a.rbtl_mags],\n",
    "    'RBTL $\\Delta M$ uncertainty': ['%.3f' % i for i in mag_errs],\n",
    "    'Twins Embedding Coordinate 1': ['%.3f' % i for i in a.embedding[:, 0]],\n",
    "    'Twins Embedding Coordinate 2': ['%.3f' % i for i in a.embedding[:, 1]],\n",
    "    'Twins Embedding Coordinate 3': ['%.3f' % i for i in a.embedding[:, 2]],\n",
    "})\n",
    "\n",
    "def dump_data(f, start=None, end=None):\n",
    "    for row in data_table[start:end]:\n",
    "        print(' & '.join(row) + ' \\\\\\\\', file=f)\n",
    "\n",
    "with a.latex_open('twins_manifold_coordinates.tex') as f:\n",
    "    dump_data(f)\n",
    "\n",
    "with a.latex_open('twins_manifold_coordinates_short.tex') as f:\n",
    "    dump_data(f, 95, 105)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dump of the spectra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "outdir = './data_release/'\n",
    "specdir = './data_release/spectra/'\n",
    "\n",
    "os.makedirs(outdir, exist_ok=True)\n",
    "os.makedirs(specdir, exist_ok=True)\n",
    "\n",
    "readme_txt = \"\"\"Data release for the Twins Embedding analysis (Boone et al. 2021a, 2021b)\n",
    "\n",
    "Contact: Kyle Boone <kyboone@uw.edu>\n",
    "\n",
    "These files contain the spectra of each of the supernovae used in the Twins Embedding analysis.\n",
    "The spectra are provided in the restframe after they have been shifted to maximum light\n",
    "and dereddened using the Reading Between the Lines (RBTL) algorithm.\n",
    "\n",
    "Each file contains the data for a single supernova, with three columns of wavelength,\n",
    "flux, and flux uncertainty.\n",
    "\"\"\"\n",
    "\n",
    "with open(f'{outdir}/README.txt', 'w') as f:\n",
    "    print(readme_txt, file=f)\n",
    "\n",
    "for idx, target in enumerate(a.targets):\n",
    "    target_name = a.iau_name_map.get(target.name, target.name)\n",
    "    target_flux = a.scale_flux[idx]\n",
    "    target_fluxerr = a.scale_fluxerr[idx]\n",
    "    with open(f'{specdir}/{target_name}.dat', 'w') as f:\n",
    "        print(' Wavelength            Flux     Uncertainty', file=f)\n",
    "        for wave, flux, fluxerr in zip(a.wave, target_flux, target_fluxerr):\n",
    "            print(f'{wave:11f} {flux:15e} {fluxerr:15e}', file=f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "toc-autonumbering": true,
  "toc-showmarkdowntxt": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}